From 30fd14a94c67d39c3b9fc04056d2bbbafae40ba3 Mon Sep 17 00:00:00 2001 From: Sergey Magidovich Date: Wed, 13 Jan 2016 16:56:27 +0300 Subject: [PATCH] Allow more than 255 bytes in metadata fields. --- generator/feature_sorter.cpp | 2 +- indexer/feature_loader.cpp | 5 +- indexer/feature_loader_base.hpp | 2 + indexer/feature_meta.hpp | 280 +++++++++--------- .../indexer_tests/feature_metadata_test.cpp | 33 +-- platform/mwm_version.hpp | 3 +- 6 files changed, 164 insertions(+), 161 deletions(-) diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index c469e8af67..cb895e4739 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -522,7 +522,7 @@ namespace feature ASSERT_LESS_OR_EQUAL(offset, numeric_limits::max(), ()); m_metadataIndex.emplace_back(ftID, static_cast(offset)); - fb.GetMetadata().SerializeToMWM(*w); + fb.GetMetadata().Serialize(*w); } uint64_t const osmID = fb.GetWayIDForRouting(); diff --git a/indexer/feature_loader.cpp b/indexer/feature_loader.cpp index fc5e5ad6e4..935c6e0e6b 100644 --- a/indexer/feature_loader.cpp +++ b/indexer/feature_loader.cpp @@ -280,7 +280,10 @@ void LoaderCurrent::ParseMetadata() { ReaderSource src(m_Info.GetMetadataReader()); src.Skip(it->value); - m_pF->GetMetadata().DeserializeFromMWM(src); + if (m_Info.GetMWMFormat() >= version::Format::v8) + m_pF->GetMetadata().Deserialize(src); + else + m_pF->GetMetadata().DeserializeFromMWMv7OrLower(src); } } catch (Reader::OpenException const &) diff --git a/indexer/feature_loader_base.hpp b/indexer/feature_loader_base.hpp index 1ed33b20d0..7245aa2da2 100644 --- a/indexer/feature_loader_base.hpp +++ b/indexer/feature_loader_base.hpp @@ -37,6 +37,8 @@ namespace feature LoaderBase * GetLoader() const { return m_pLoader; } + inline version::Format GetMWMFormat() const { return m_header.GetFormat(); } + inline serial::CodingParams const & GetDefCodingParams() const { return m_header.GetDefCodingParams(); diff --git a/indexer/feature_meta.hpp b/indexer/feature_meta.hpp index ff798a6c83..97dc5e2d9d 100644 --- a/indexer/feature_meta.hpp +++ b/indexer/feature_meta.hpp @@ -12,154 +12,158 @@ namespace feature { - class MetadataBase +class MetadataBase +{ +protected: + // TODO: Change uint8_t to appropriate type when FMD_COUNT reaches 256. + void Set(uint8_t type, string const & value) { - protected: - void Set(uint8_t type, string const & value) + auto found = m_metadata.find(type); + if (found == m_metadata.end()) { - auto found = m_metadata.find(type); - if (found == m_metadata.end()) - { - if (!value.empty()) - m_metadata[type] = value; - } + if (!value.empty()) + m_metadata[type] = value; + } + else + { + if (value.empty()) + m_metadata.erase(found); else - { - if (value.empty()) - m_metadata.erase(found); - else - found->second = value; - } + found->second = value; } + } - public: - string Get(uint8_t type) const - { - auto it = m_metadata.find(type); - return (it == m_metadata.end()) ? string() : it->second; - } - - vector GetPresentTypes() const - { - vector types; - types.reserve(m_metadata.size()); - - for (auto const & item : m_metadata) - types.push_back(item.first); - - return types; - } - - inline bool Empty() const { return m_metadata.empty(); } - inline size_t Size() const { return m_metadata.size(); } - - template void Serialize(TSink & sink) const - { - uint8_t const sz = m_metadata.size(); - WriteToSink(sink, sz); - for (auto const & it : m_metadata) - { - WriteToSink(sink, static_cast(it.first)); - utils::WriteString(sink, it.second); - } - } - - template void Deserialize(TSource & src) - { - uint8_t const sz = ReadPrimitiveFromSource(src); - for (size_t i = 0; i < sz; ++i) - { - uint8_t const key = ReadPrimitiveFromSource(src); - string value; - utils::ReadString(src, value); - m_metadata[key].swap(value); - } - } - - protected: - map m_metadata; - }; - - class Metadata : public MetadataBase +public: + string Get(uint8_t type) const { - public: - /// @note! Do not change values here. - /// Add new types to the end of list, before FMD_COUNT. - enum EType - { - FMD_CUISINE = 1, - FMD_OPEN_HOURS = 2, - FMD_PHONE_NUMBER = 3, - FMD_FAX_NUMBER = 4, - FMD_STARS = 5, - FMD_OPERATOR = 6, - FMD_URL = 7, - FMD_WEBSITE = 8, - FMD_INTERNET = 9, - FMD_ELE = 10, - FMD_TURN_LANES = 11, - FMD_TURN_LANES_FORWARD = 12, - FMD_TURN_LANES_BACKWARD = 13, - FMD_EMAIL = 14, - FMD_POSTCODE = 15, - FMD_WIKIPEDIA = 16, - FMD_MAXSPEED = 17, - FMD_FLATS = 18, - FMD_HEIGHT = 19, - FMD_MIN_HEIGHT = 20, - FMD_DENOMINATION = 21, - FMD_COUNT - }; + auto it = m_metadata.find(type); + return (it == m_metadata.end()) ? string() : it->second; + } - static_assert(FMD_COUNT <= 255, "Meta types count is limited to one byte."); - - void Set(EType type, string const & value) - { - MetadataBase::Set(type, value); - } - - void Drop(EType type) { Set(type, string()); } - - string GetWikiURL() const; - - template void SerializeToMWM(TWriter & writer) const - { - for (auto const & e : m_metadata) - { - // Set high bit if it's the last element. - uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0); - uint8_t elem[2] = {static_cast(e.first | mark), - static_cast(min(e.second.size(), (size_t)kMaxStringLength))}; - writer.Write(elem, sizeof(elem)); - writer.Write(e.second.data(), elem[1]); - } - } - - template void DeserializeFromMWM(TSource & src) - { - uint8_t header[2] = {0}; - char buffer[kMaxStringLength] = {0}; - do - { - src.Read(header, sizeof(header)); - src.Read(buffer, header[1]); - m_metadata[header[0] & 0x7F].assign(buffer, header[1]); - } while (!(header[0] & 0x80)); - } - - private: - enum { kMaxStringLength = 255 }; - }; - - class AddressData : public MetadataBase + vector GetPresentTypes() const { - public: - enum Type { PLACE, STREET, POSTCODE }; + vector types; + types.reserve(m_metadata.size()); - void Add(Type type, string const & s) + for (auto const & item : m_metadata) + types.push_back(item.first); + + return types; + } + + inline bool Empty() const { return m_metadata.empty(); } + inline size_t Size() const { return m_metadata.size(); } + + template + void Serialize(TSink & sink) const + { + auto const sz = static_cast(m_metadata.size()); + WriteVarUint(sink, sz); + for (auto const & it : m_metadata) { - /// @todo Probably, we need to add separator here and store multiple values. - MetadataBase::Set(type, s); + WriteVarUint(sink, static_cast(it.first)); + utils::WriteString(sink, it.second); } + } + + template + void Deserialize(TSource & src) + { + auto const sz = ReadVarUint(src); + for (size_t i = 0; i < sz; ++i) + { + auto const key = ReadVarUint(src); + utils::ReadString(src, m_metadata[key]); + } + } + +protected: + map m_metadata; +}; + +class Metadata : public MetadataBase +{ +public: + /// @note! Do not change values here. + /// Add new types to the end of list, before FMD_COUNT. + enum EType + { + FMD_CUISINE = 1, + FMD_OPEN_HOURS = 2, + FMD_PHONE_NUMBER = 3, + FMD_FAX_NUMBER = 4, + FMD_STARS = 5, + FMD_OPERATOR = 6, + FMD_URL = 7, + FMD_WEBSITE = 8, + FMD_INTERNET = 9, + FMD_ELE = 10, + FMD_TURN_LANES = 11, + FMD_TURN_LANES_FORWARD = 12, + FMD_TURN_LANES_BACKWARD = 13, + FMD_EMAIL = 14, + FMD_POSTCODE = 15, + FMD_WIKIPEDIA = 16, + FMD_MAXSPEED = 17, + FMD_FLATS = 18, + FMD_HEIGHT = 19, + FMD_MIN_HEIGHT = 20, + FMD_DENOMINATION = 21, + FMD_COUNT }; + + void Set(EType type, string const & value) + { + MetadataBase::Set(type, value); + } + + void Drop(EType type) { Set(type, string()); } + + string GetWikiURL() const; + + // TODO: Commented code below is now longer neded, but I leave it here + // as a hint to what is going on in DeserializeFromMWMv7OrLower. + // Please, remove it when DeserializeFromMWMv7OrLower is no longer neded. + // template + // void SerializeToMWM(TWriter & writer) const + // { + // for (auto const & e : m_metadata) + // { + // // Set high bit if it's the last element. + // uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0); + // uint8_t elem[2] = {static_cast(e.first | mark), + // static_cast(min(e.second.size(), (size_t)kMaxStringLength))}; + // writer.Write(elem, sizeof(elem)); + // writer.Write(e.second.data(), elem[1]); + // } + // } + + template + void DeserializeFromMWMv7OrLower(TSource & src) + { + uint8_t header[2] = {0}; + char buffer[kMaxStringLength] = {0}; + do + { + src.Read(header, sizeof(header)); + src.Read(buffer, header[1]); + m_metadata[header[0] & 0x7F].assign(buffer, header[1]); + } while (!(header[0] & 0x80)); + } + +private: + enum { kMaxStringLength = 255 }; +}; + +class AddressData : public MetadataBase +{ +public: + enum Type { PLACE, STREET, POSTCODE }; + + void Add(Type type, string const & s) + { + /// @todo Probably, we need to add separator here and store multiple values. + MetadataBase::Set(type, s); + } +}; } diff --git a/indexer/indexer_tests/feature_metadata_test.cpp b/indexer/indexer_tests/feature_metadata_test.cpp index 9b2cdb9d16..1a1771c634 100644 --- a/indexer/indexer_tests/feature_metadata_test.cpp +++ b/indexer/indexer_tests/feature_metadata_test.cpp @@ -16,7 +16,19 @@ map const kKeyValues = { {Metadata::FMD_ELE, "12345"}, {Metadata::FMD_CUISINE, "greek;mediterranean"}, - {Metadata::FMD_EMAIL, "cool@email.at"} + {Metadata::FMD_EMAIL, "cool@email.at"}, + // This string is longer than 255 bytes. + {Metadata::FMD_URL, "http://rskxmkjwnikfnjqhyv" + "kpjgaghhyhukjyenduiuanxgb" + "mndtlpfphdgaizfcpzuiuspcp" + "umeojwvekvjprlutwjmxudyzr" + "lwwsepewevsuqelobqcfdzsoq" + "ozkesghojribepbaitivmaqep" + "hheckitonddqhbapdybhetvnw" + "vlchjafepdjaeoaapysdvculx" + "uwjbgdddryodiihvnpvmkgqvs" + "mawbdsrbmnndcozmrgeoahbkh" + "cevxkmtdqnxpxlsju.org"} }; } // namespace @@ -74,25 +86,6 @@ UNIT_TEST(Feature_Serialization) TEST_EQUAL(serialized.Get(Metadata::FMD_OPERATOR), "", ()); TEST_EQUAL(serialized.Size(), kKeyValues.size(), ()); } - - { - Metadata serialized; - vector buffer; - MemWriter writer(buffer); - // Here is the difference. - original.SerializeToMWM(writer); - - MemReader reader(buffer.data(), buffer.size()); - ReaderSource src(reader); - // Here is another difference. - serialized.DeserializeFromMWM(src); - - for (auto const & value : kKeyValues) - TEST_EQUAL(serialized.Get(value.first), value.second, ()); - - TEST_EQUAL(serialized.Get(Metadata::FMD_OPERATOR), "", ()); - TEST_EQUAL(serialized.Size(), kKeyValues.size(), ()); - } } UNIT_TEST(Feature_Metadata_GetWikipedia) diff --git a/platform/mwm_version.hpp b/platform/mwm_version.hpp index 7cc2b8f32a..c49a17fa6d 100644 --- a/platform/mwm_version.hpp +++ b/platform/mwm_version.hpp @@ -19,7 +19,8 @@ enum class Format v5, // July 2015 (feature id is the index in vector now). v6, // October 2015 (offsets vector is in mwm now). v7, // November 2015 (supply different search index formats). - lastFormat = v7 + v8, // January 2016 (long strings in metadata). + lastFormat = v8 }; struct MwmVersion