pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Allow more than 255 bytes in metadata fields.

Browse source
This commit is contained in:
Sergey Magidovich 2016-01-13 16:56:27 +03:00 • committed by Sergey Yershov
parent 7da95d1f1c
commit 30fd14a94c
6 changed files with 164 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
generator/feature_sorter.cpp
View file

@ -522,7 +522,7 @@ namespace feature
ASSERT_LESS_OR_EQUAL(offset, numeric_limits<uint32_t>::max(), ());
m_metadataIndex.emplace_back(ftID, static_cast<uint32_t>(offset));
fb.GetMetadata().SerializeToMWM(*w);
fb.GetMetadata().Serialize(*w);
}
uint64_t const osmID = fb.GetWayIDForRouting();

5
indexer/feature_loader.cpp
View file

@ -280,7 +280,10 @@ void LoaderCurrent::ParseMetadata()
{
ReaderSource<FilesContainerR::ReaderT> src(m_Info.GetMetadataReader());
src.Skip(it->value);
m_pF->GetMetadata().DeserializeFromMWM(src);
if (m_Info.GetMWMFormat() >= version::Format::v8)
m_pF->GetMetadata().Deserialize(src);
else
m_pF->GetMetadata().DeserializeFromMWMv7OrLower(src);
}
}
catch (Reader::OpenException const &)

2
indexer/feature_loader_base.hpp
View file

@ -37,6 +37,8 @@ namespace feature
LoaderBase * GetLoader() const { return m_pLoader; }
inline version::Format GetMWMFormat() const { return m_header.GetFormat(); }
inline serial::CodingParams const & GetDefCodingParams() const
{
return m_header.GetDefCodingParams();

280
indexer/feature_meta.hpp
View file

@ -12,154 +12,158 @@
namespace feature
{
class MetadataBase
class MetadataBase
{
protected:
// TODO: Change uint8_t to appropriate type when FMD_COUNT reaches 256.
void Set(uint8_t type, string const & value)
{
protected:
void Set(uint8_t type, string const & value)
auto found = m_metadata.find(type);
if (found == m_metadata.end())
{
auto found = m_metadata.find(type);
if (found == m_metadata.end())
{
if (!value.empty())
m_metadata[type] = value;
}
if (!value.empty())
m_metadata[type] = value;
}
else
{
if (value.empty())
m_metadata.erase(found);
else
{
if (value.empty())
m_metadata.erase(found);
else
found->second = value;
}
found->second = value;
}
}
public:
string Get(uint8_t type) const
{
auto it = m_metadata.find(type);
return (it == m_metadata.end()) ? string() : it->second;
}
vector<uint8_t> GetPresentTypes() const
{
vector<uint8_t> types;
types.reserve(m_metadata.size());
for (auto const & item : m_metadata)
types.push_back(item.first);
return types;
}
inline bool Empty() const { return m_metadata.empty(); }
inline size_t Size() const { return m_metadata.size(); }
template <class TSink> void Serialize(TSink & sink) const
{
uint8_t const sz = m_metadata.size();
WriteToSink(sink, sz);
for (auto const & it : m_metadata)
{
WriteToSink(sink, static_cast<uint8_t>(it.first));
utils::WriteString(sink, it.second);
}
}
template <class TSource> void Deserialize(TSource & src)
{
uint8_t const sz = ReadPrimitiveFromSource<uint8_t>(src);
for (size_t i = 0; i < sz; ++i)
{
uint8_t const key = ReadPrimitiveFromSource<uint8_t>(src);
string value;
utils::ReadString(src, value);
m_metadata[key].swap(value);
}
}
protected:
map<uint8_t, string> m_metadata;
};
class Metadata : public MetadataBase
public:
string Get(uint8_t type) const
{
public:
/// @note! Do not change values here.
/// Add new types to the end of list, before FMD_COUNT.
enum EType
{
FMD_CUISINE = 1,
FMD_OPEN_HOURS = 2,
FMD_PHONE_NUMBER = 3,
FMD_FAX_NUMBER = 4,
FMD_STARS = 5,
FMD_OPERATOR = 6,
FMD_URL = 7,
FMD_WEBSITE = 8,
FMD_INTERNET = 9,
FMD_ELE = 10,
FMD_TURN_LANES = 11,
FMD_TURN_LANES_FORWARD = 12,
FMD_TURN_LANES_BACKWARD = 13,
FMD_EMAIL = 14,
FMD_POSTCODE = 15,
FMD_WIKIPEDIA = 16,
FMD_MAXSPEED = 17,
FMD_FLATS = 18,
FMD_HEIGHT = 19,
FMD_MIN_HEIGHT = 20,
FMD_DENOMINATION = 21,
FMD_COUNT
};
auto it = m_metadata.find(type);
return (it == m_metadata.end()) ? string() : it->second;
}
static_assert(FMD_COUNT <= 255, "Meta types count is limited to one byte.");
void Set(EType type, string const & value)
{
MetadataBase::Set(type, value);
}
void Drop(EType type) { Set(type, string()); }
string GetWikiURL() const;
template <class TWriter> void SerializeToMWM(TWriter & writer) const
{
for (auto const & e : m_metadata)
{
// Set high bit if it's the last element.
uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0);
uint8_t elem[2] = {static_cast<uint8_t>(e.first | mark),
static_cast<uint8_t>(min(e.second.size(), (size_t)kMaxStringLength))};
writer.Write(elem, sizeof(elem));
writer.Write(e.second.data(), elem[1]);
}
}
template <class TSource> void DeserializeFromMWM(TSource & src)
{
uint8_t header[2] = {0};
char buffer[kMaxStringLength] = {0};
do
{
src.Read(header, sizeof(header));
src.Read(buffer, header[1]);
m_metadata[header[0] & 0x7F].assign(buffer, header[1]);
} while (!(header[0] & 0x80));
}
private:
enum { kMaxStringLength = 255 };
};
class AddressData : public MetadataBase
vector<uint8_t> GetPresentTypes() const
{
public:
enum Type { PLACE, STREET, POSTCODE };
vector<uint8_t> types;
types.reserve(m_metadata.size());
void Add(Type type, string const & s)
for (auto const & item : m_metadata)
types.push_back(item.first);
return types;
}
inline bool Empty() const { return m_metadata.empty(); }
inline size_t Size() const { return m_metadata.size(); }
template <class TSink>
void Serialize(TSink & sink) const
{
auto const sz = static_cast<uint32_t>(m_metadata.size());
WriteVarUint(sink, sz);
for (auto const & it : m_metadata)
{
/// @todo Probably, we need to add separator here and store multiple values.
MetadataBase::Set(type, s);
WriteVarUint(sink, static_cast<uint32_t>(it.first));
utils::WriteString(sink, it.second);
}
}
template <class TSource>
void Deserialize(TSource & src)
{
auto const sz = ReadVarUint<uint32_t>(src);
for (size_t i = 0; i < sz; ++i)
{
auto const key = ReadVarUint<uint32_t>(src);
utils::ReadString(src, m_metadata[key]);
}
}
protected:
map<uint8_t, string> m_metadata;
};
class Metadata : public MetadataBase
{
public:
/// @note! Do not change values here.
/// Add new types to the end of list, before FMD_COUNT.
enum EType
{
FMD_CUISINE = 1,
FMD_OPEN_HOURS = 2,
FMD_PHONE_NUMBER = 3,
FMD_FAX_NUMBER = 4,
FMD_STARS = 5,
FMD_OPERATOR = 6,
FMD_URL = 7,
FMD_WEBSITE = 8,
FMD_INTERNET = 9,
FMD_ELE = 10,
FMD_TURN_LANES = 11,
FMD_TURN_LANES_FORWARD = 12,
FMD_TURN_LANES_BACKWARD = 13,
FMD_EMAIL = 14,
FMD_POSTCODE = 15,
FMD_WIKIPEDIA = 16,
FMD_MAXSPEED = 17,
FMD_FLATS = 18,
FMD_HEIGHT = 19,
FMD_MIN_HEIGHT = 20,
FMD_DENOMINATION = 21,
FMD_COUNT
};
void Set(EType type, string const & value)
{
MetadataBase::Set(type, value);
}
void Drop(EType type) { Set(type, string()); }
string GetWikiURL() const;
// TODO: Commented code below is now longer neded, but I leave it here
// as a hint to what is going on in DeserializeFromMWMv7OrLower.
// Please, remove it when DeserializeFromMWMv7OrLower is no longer neded.
// template <class TWriter>
// void SerializeToMWM(TWriter & writer) const
// {
// for (auto const & e : m_metadata)
// {
// // Set high bit if it's the last element.
// uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0);
// uint8_t elem[2] = {static_cast<uint8_t>(e.first | mark),
// static_cast<uint8_t>(min(e.second.size(), (size_t)kMaxStringLength))};
// writer.Write(elem, sizeof(elem));
// writer.Write(e.second.data(), elem[1]);
// }
// }
template <class TSource>
void DeserializeFromMWMv7OrLower(TSource & src)
{
uint8_t header[2] = {0};
char buffer[kMaxStringLength] = {0};
do
{
src.Read(header, sizeof(header));
src.Read(buffer, header[1]);
m_metadata[header[0] & 0x7F].assign(buffer, header[1]);
} while (!(header[0] & 0x80));
}
private:
enum { kMaxStringLength = 255 };
};
class AddressData : public MetadataBase
{
public:
enum Type { PLACE, STREET, POSTCODE };
void Add(Type type, string const & s)
{
/// @todo Probably, we need to add separator here and store multiple values.
MetadataBase::Set(type, s);
}
};
}

33
indexer/indexer_tests/feature_metadata_test.cpp
View file

@ -16,7 +16,19 @@ map<Metadata::EType, string> const kKeyValues =
{
{Metadata::FMD_ELE, "12345"},
{Metadata::FMD_CUISINE, "greek;mediterranean"},
{Metadata::FMD_EMAIL, "cool@email.at"}
{Metadata::FMD_EMAIL, "cool@email.at"},
// This string is longer than 255 bytes.
{Metadata::FMD_URL, "http://rskxmkjwnikfnjqhyv"
"kpjgaghhyhukjyenduiuanxgb"
"mndtlpfphdgaizfcpzuiuspcp"
"umeojwvekvjprlutwjmxudyzr"
"lwwsepewevsuqelobqcfdzsoq"
"ozkesghojribepbaitivmaqep"
"hheckitonddqhbapdybhetvnw"
"vlchjafepdjaeoaapysdvculx"
"uwjbgdddryodiihvnpvmkgqvs"
"mawbdsrbmnndcozmrgeoahbkh"
"cevxkmtdqnxpxlsju.org"}
};
} // namespace
@ -74,25 +86,6 @@ UNIT_TEST(Feature_Serialization)
TEST_EQUAL(serialized.Get(Metadata::FMD_OPERATOR), "", ());
TEST_EQUAL(serialized.Size(), kKeyValues.size(), ());
}
{
Metadata serialized;
vector<char> buffer;
MemWriter<decltype(buffer)> writer(buffer);
// Here is the difference.
original.SerializeToMWM(writer);
MemReader reader(buffer.data(), buffer.size());
ReaderSource<MemReader> src(reader);
// Here is another difference.
serialized.DeserializeFromMWM(src);
for (auto const & value : kKeyValues)
TEST_EQUAL(serialized.Get(value.first), value.second, ());
TEST_EQUAL(serialized.Get(Metadata::FMD_OPERATOR), "", ());
TEST_EQUAL(serialized.Size(), kKeyValues.size(), ());
}
}
UNIT_TEST(Feature_Metadata_GetWikipedia)

3
platform/mwm_version.hpp
View file

@ -19,7 +19,8 @@ enum class Format
v5, // July 2015 (feature id is the index in vector now).
v6, // October 2015 (offsets vector is in mwm now).
v7, // November 2015 (supply different search index formats).
lastFormat = v7
v8, // January 2016 (long strings in metadata).
lastFormat = v8
};
struct MwmVersion