diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index 72b6dc181b..64c9315f0c 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -93,12 +93,8 @@ namespace feature unique_ptr m_metadataWriter; unique_ptr m_searchTokensWriter; - struct TMetadataIndexEntry - { - uint32_t key; - uint32_t value; - }; - vector m_metadataIndex; + // Mapping from feature id to offset in file section with the correspondent metadata. + vector> m_metadataIndex; DataHeader m_header; uint32_t m_versionDate; @@ -107,11 +103,11 @@ namespace feature public: FeaturesCollector2(string const & fName, DataHeader const & header, uint32_t versionDate) - : FeaturesCollector(fName + DATA_FILE_TAG), m_writer(fName), m_header(header), m_versionDate(versionDate) + : FeaturesCollector(fName + DATA_FILE_TAG), m_writer(fName), + m_metadataWriter(new FileWriter(fName + METADATA_FILE_TAG)), + m_searchTokensWriter(new FileWriter(fName + SEARCH_TOKENS_FILE_TAG)), + m_header(header), m_versionDate(versionDate) { - m_metadataWriter.reset(new FileWriter(fName + METADATA_FILE_TAG)); - m_searchTokensWriter.reset(new FileWriter(fName + SEARCH_TOKENS_FILE_TAG)); - for (size_t i = 0; i < m_header.GetScalesCount(); ++i) { string const postfix = strings::to_string(i); @@ -163,11 +159,12 @@ namespace feature } { + /// @todo Replace this mapping vector with succint structure. FileWriter w = m_writer.GetWriter(METADATA_INDEX_FILE_TAG); for (auto const & v : m_metadataIndex) { - WriteToSink(w, v.key); - WriteToSink(w, v.value); + WriteToSink(w, v.first); + WriteToSink(w, v.second); } } @@ -516,9 +513,9 @@ namespace feature if (!fb.GetMetadata().Empty()) { - uint64_t offset = m_metadataWriter->Pos(); + uint64_t const offset = m_metadataWriter->Pos(); ASSERT_LESS_OR_EQUAL(offset, numeric_limits::max(), ()); - m_metadataIndex.push_back({ ftID, static_cast(offset) }); + m_metadataIndex.emplace_back(ftID, static_cast(offset)); fb.GetMetadata().SerializeToMWM(*m_metadataWriter); } diff --git a/generator/generator_tests/feature_builder_test.cpp b/generator/generator_tests/feature_builder_test.cpp index 6192a11e2d..c53fcbaa4b 100644 --- a/generator/generator_tests/feature_builder_test.cpp +++ b/generator/generator_tests/feature_builder_test.cpp @@ -155,7 +155,7 @@ UNIT_TEST(FeatureParams_Parsing) { FeatureParams params; - params.AddStreet("Embarcadero street \t\t 85"); + params.AddStreet("Embarcadero\nstreet"); TEST_EQUAL(params.GetStreet(), "Embarcadero street", ()); } diff --git a/generator/osm_translator.hpp b/generator/osm_translator.hpp index 2e06c4d085..43d3d102aa 100644 --- a/generator/osm_translator.hpp +++ b/generator/osm_translator.hpp @@ -141,8 +141,11 @@ protected: for (auto const & p : e.tags) { - // Store only this tags to use it in railway stations processing for the particular city. - if (p.first == "network" || p.first == "operator" || p.first == "route" || p.first == "maxspeed" || + // - used in railway station processing + // - used in routing information + // - used in building addresses matching + if (p.first == "network" || p.first == "operator" || p.first == "route" || + p.first == "maxspeed" || strings::StartsWith(p.first, "addr:")) { if (!TBase::IsKeyTagExists(p.first)) diff --git a/indexer/feature_data.cpp b/indexer/feature_data.cpp index a0bf7e1091..9dd042da2c 100644 --- a/indexer/feature_data.cpp +++ b/indexer/feature_data.cpp @@ -164,11 +164,6 @@ bool IsDummyName(string const & s) s == "Edificio" || s == "edificio"); } -struct IsBadChar -{ - bool operator() (char c) const { return (c == '\n'); } -}; - } ///////////////////////////////////////////////////////////////////////////////////////// @@ -224,19 +219,10 @@ bool FeatureParams::AddHouseNumber(string const & ss) void FeatureParams::AddStreet(string s) { - // Erase bad chars (\n) because we write addresses to txt file. - s.erase(remove_if(s.begin(), s.end(), IsBadChar()), s.end()); + // Replace \n with spaces because we write addresses to txt file. + replace(s.begin(), s.end(), '\n', ' '); - // Osm likes to put house numbers into addr:street field. - size_t i = s.find_last_of("\t "); - if (i != string::npos) - { - uint64_t n; - if (strings::to_uint64(s.substr(i+1), n)) - s.erase(s.find_last_not_of("\t ", i)+1); - } - - m_addrTags.Add(AddressData::FAD_STREET, s); + m_addrTags.Add(AddressData::STREET, s); } void FeatureParams::AddAddress(string const & s) @@ -251,22 +237,26 @@ void FeatureParams::AddAddress(string const & s) i = s.find_first_not_of("\t ", i); } else + { i = 0; + } } else + { i = 0; + } - AddStreet(s.substr(i, s.size()-i)); + AddStreet(s.substr(i, s.size() - i)); } void FeatureParams::AddPlace(string const & s) { - m_addrTags.Add(AddressData::FAD_PLACE, s); + m_addrTags.Add(AddressData::PLACE, s); } void FeatureParams::AddPostcode(string const & s) { - m_addrTags.Add(AddressData::FAD_POSTCODE, s); + m_addrTags.Add(AddressData::POSTCODE, s); } bool FeatureParams::FormatFullAddress(m2::PointD const & pt, string & res) const @@ -285,7 +275,7 @@ bool FeatureParams::FormatFullAddress(m2::PointD const & pt, string & res) const string FeatureParams::GetStreet() const { - return m_addrTags.Get(AddressData::FAD_STREET); + return m_addrTags.Get(AddressData::STREET); } void FeatureParams::SetGeomType(feature::EGeomType t) diff --git a/indexer/feature_data.hpp b/indexer/feature_data.hpp index c12b309ea0..c1de7d4c4d 100644 --- a/indexer/feature_data.hpp +++ b/indexer/feature_data.hpp @@ -274,6 +274,9 @@ public: feature::Metadata const & GetMetadata() const { return m_metadata; } feature::Metadata & GetMetadata() { return m_metadata; } + /// @param[in] fullStoring \n + /// - true when saving in temporary files after first generation step \n + /// - false when final mwm saving template void Write(SinkT & sink, bool fullStoring) const { uint8_t const header = GetHeader(); diff --git a/indexer/feature_loader.cpp b/indexer/feature_loader.cpp index 23c69645a1..fc5e5ad6e4 100644 --- a/indexer/feature_loader.cpp +++ b/indexer/feature_loader.cpp @@ -278,9 +278,9 @@ void LoaderCurrent::ParseMetadata() if (it != idx.end() && m_pF->m_id.m_index == it->key) { - ReaderSource reader(m_Info.GetMetadataReader()); - reader.Skip(it->value); - m_pF->GetMetadata().DeserializeFromMWM(reader); + ReaderSource src(m_Info.GetMetadataReader()); + src.Skip(it->value); + m_pF->GetMetadata().DeserializeFromMWM(src); } } catch (Reader::OpenException const &) diff --git a/indexer/feature_meta.hpp b/indexer/feature_meta.hpp index eda4527c84..0faaabc571 100644 --- a/indexer/feature_meta.hpp +++ b/indexer/feature_meta.hpp @@ -32,34 +32,29 @@ namespace feature return types; } - void Drop(uint8_t type) - { - m_metadata.erase(type); - } - inline bool Empty() const { return m_metadata.empty(); } inline size_t Size() const { return m_metadata.size(); } - template void Serialize(ArchiveT & ar) const + template void Serialize(TSink & sink) const { uint8_t const sz = m_metadata.size(); - WriteToSink(ar, sz); + WriteToSink(sink, sz); for (auto const & it : m_metadata) { - WriteToSink(ar, static_cast(it.first)); - utils::WriteString(ar, it.second); + WriteToSink(sink, static_cast(it.first)); + utils::WriteString(sink, it.second); } } - template void Deserialize(ArchiveT & ar) + template void Deserialize(TSource & src) { - uint8_t const sz = ReadPrimitiveFromSource(ar); + uint8_t const sz = ReadPrimitiveFromSource(src); for (size_t i = 0; i < sz; ++i) { - uint8_t const key = ReadPrimitiveFromSource(ar); + uint8_t const key = ReadPrimitiveFromSource(src); string value; - utils::ReadString(ar, value); - m_metadata.insert(make_pair(key, value)); + utils::ReadString(src, value); + m_metadata[key].swap(value); } } @@ -100,7 +95,7 @@ namespace feature static_assert(FMD_COUNT <= 255, "Meta types count is limited to one byte."); - void Add(EType type, string const & s) + void Set(EType type, string const & value) { auto found = m_metadata.find(type); if (found == m_metadata.end()) @@ -117,40 +112,11 @@ namespace feature } } - /// Synonym of Set(type, ""). - void Drop(EType type) - { - Set(type, ""); - } - - string Get(EType type) const - { - auto const it = m_metadata.find(type); - return (it == m_metadata.end()) ? string() : it->second; - } - - vector GetPresentTypes() const - { - vector types; - types.reserve(m_metadata.size()); - - for (auto const & item : m_metadata) - types.push_back(item.first); - - return types; - } - - inline bool Empty() const { return m_metadata.empty(); } - inline size_t Size() const { return m_metadata.size(); } + void Drop(EType type) { Set(type, ""); } string GetWikiURL() const; -======= - val = val + ", " + s; - } - ->>>>>>> 12268b5... [generator] Pass address tokens to the search index generation step. - template void SerializeToMWM(ArchiveT & ar) const + template void SerializeToMWM(TWriter & writer) const { for (auto const & e : m_metadata) { @@ -158,19 +124,19 @@ namespace feature uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0); uint8_t elem[2] = {static_cast(e.first | mark), static_cast(min(e.second.size(), (size_t)kMaxStringLength))}; - ar.Write(elem, sizeof(elem)); - ar.Write(e.second.data(), elem[1]); + writer.Write(elem, sizeof(elem)); + writer.Write(e.second.data(), elem[1]); } } - template void DeserializeFromMWM(ArchiveT & ar) + template void DeserializeFromMWM(TSource & src) { uint8_t header[2] = {0}; char buffer[kMaxStringLength] = {0}; do { - ar.Read(header, sizeof(header)); - ar.Read(buffer, header[1]); + src.Read(header, sizeof(header)); + src.Read(buffer, header[1]); m_metadata[header[0] & 0x7F].assign(buffer, header[1]); } while (!(header[0] & 0x80)); } @@ -182,14 +148,9 @@ namespace feature class AddressData : public MetadataBase { public: - enum EType - { - FAD_PLACE = 1, - FAD_STREET = 2, - FAD_POSTCODE = 3, - }; + enum Type { PLACE, STREET, POSTCODE }; - void Add(EType type, string const & s) + void Add(Type type, string const & s) { /// @todo Probably, we need to add separator here and store multiple values. m_metadata[type] = s; diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index 1b3be46766..75d83135d0 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -286,7 +286,7 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest & features, CategoriesHolder & inserter.m_val.m_featureId = index++; - string const street = data.Get(feature::AddressData::FAD_STREET); + string const street = data.Get(feature::AddressData::STREET); if (!street.empty()) inserter(lang, street); }