pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Review fixes.

Browse source
This commit is contained in:
vng 2015-11-18 13:17:22 +03:00 • committed by Sergey Yershov
parent 330aee1838
commit 46fbd59cc4
8 changed files with 54 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
generator/feature_sorter.cpp
View file

@ -93,12 +93,8 @@ namespace feature
unique_ptr<FileWriter> m_metadataWriter;
unique_ptr<FileWriter> m_searchTokensWriter;
struct TMetadataIndexEntry
{
uint32_t key;
uint32_t value;
};
vector<TMetadataIndexEntry> m_metadataIndex;
// Mapping from feature id to offset in file section with the correspondent metadata.
vector<pair<uint32_t, uint32_t>> m_metadataIndex;
DataHeader m_header;
uint32_t m_versionDate;
@ -107,11 +103,11 @@ namespace feature
public:
FeaturesCollector2(string const & fName, DataHeader const & header, uint32_t versionDate)
: FeaturesCollector(fName + DATA_FILE_TAG), m_writer(fName), m_header(header), m_versionDate(versionDate)
: FeaturesCollector(fName + DATA_FILE_TAG), m_writer(fName),
m_metadataWriter(new FileWriter(fName + METADATA_FILE_TAG)),
m_searchTokensWriter(new FileWriter(fName + SEARCH_TOKENS_FILE_TAG)),
m_header(header), m_versionDate(versionDate)
{
m_metadataWriter.reset(new FileWriter(fName + METADATA_FILE_TAG));
m_searchTokensWriter.reset(new FileWriter(fName + SEARCH_TOKENS_FILE_TAG));
for (size_t i = 0; i < m_header.GetScalesCount(); ++i)
{
string const postfix = strings::to_string(i);
@ -163,11 +159,12 @@ namespace feature
}
{
/// @todo Replace this mapping vector with succint structure.
FileWriter w = m_writer.GetWriter(METADATA_INDEX_FILE_TAG);
for (auto const & v : m_metadataIndex)
{
WriteToSink(w, v.key);
WriteToSink(w, v.value);
WriteToSink(w, v.first);
WriteToSink(w, v.second);
}
}
@ -516,9 +513,9 @@ namespace feature
if (!fb.GetMetadata().Empty())
{
uint64_t offset = m_metadataWriter->Pos();
uint64_t const offset = m_metadataWriter->Pos();
ASSERT_LESS_OR_EQUAL(offset, numeric_limits<uint32_t>::max(), ());
m_metadataIndex.push_back({ ftID, static_cast<uint32_t>(offset) });
m_metadataIndex.emplace_back(ftID, static_cast<uint32_t>(offset));
fb.GetMetadata().SerializeToMWM(*m_metadataWriter);
}

2
generator/generator_tests/feature_builder_test.cpp
View file

@ -155,7 +155,7 @@ UNIT_TEST(FeatureParams_Parsing)
{
FeatureParams params;
params.AddStreet("Embarcadero street \t\t 85");
params.AddStreet("Embarcadero\nstreet");
TEST_EQUAL(params.GetStreet(), "Embarcadero street", ());
}

7
generator/osm_translator.hpp
View file

@ -141,8 +141,11 @@ protected:
for (auto const & p : e.tags)
{
// Store only this tags to use it in railway stations processing for the particular city.
if (p.first == "network" || p.first == "operator" || p.first == "route" || p.first == "maxspeed" ||
// - used in railway station processing
// - used in routing information
// - used in building addresses matching
if (p.first == "network" || p.first == "operator" || p.first == "route" ||
p.first == "maxspeed" ||
strings::StartsWith(p.first, "addr:"))
{
if (!TBase::IsKeyTagExists(p.first))

32
indexer/feature_data.cpp
View file

@ -164,11 +164,6 @@ bool IsDummyName(string const & s)
s == "Edificio" || s == "edificio");
}
struct IsBadChar
{
bool operator() (char c) const { return (c == '\n'); }
};
}
/////////////////////////////////////////////////////////////////////////////////////////
@ -224,19 +219,10 @@ bool FeatureParams::AddHouseNumber(string const & ss)
void FeatureParams::AddStreet(string s)
{
// Erase bad chars (\n) because we write addresses to txt file.
s.erase(remove_if(s.begin(), s.end(), IsBadChar()), s.end());
// Replace \n with spaces because we write addresses to txt file.
replace(s.begin(), s.end(), '\n', ' ');
// Osm likes to put house numbers into addr:street field.
size_t i = s.find_last_of("\t ");
if (i != string::npos)
{
uint64_t n;
if (strings::to_uint64(s.substr(i+1), n))
s.erase(s.find_last_not_of("\t ", i)+1);
}
m_addrTags.Add(AddressData::FAD_STREET, s);
m_addrTags.Add(AddressData::STREET, s);
}
void FeatureParams::AddAddress(string const & s)
@ -251,22 +237,26 @@ void FeatureParams::AddAddress(string const & s)
i = s.find_first_not_of("\t ", i);
}
else
{
i = 0;
}
}
else
{
i = 0;
}
AddStreet(s.substr(i, s.size()-i));
AddStreet(s.substr(i, s.size() - i));
}
void FeatureParams::AddPlace(string const & s)
{
m_addrTags.Add(AddressData::FAD_PLACE, s);
m_addrTags.Add(AddressData::PLACE, s);
}
void FeatureParams::AddPostcode(string const & s)
{
m_addrTags.Add(AddressData::FAD_POSTCODE, s);
m_addrTags.Add(AddressData::POSTCODE, s);
}
bool FeatureParams::FormatFullAddress(m2::PointD const & pt, string & res) const
@ -285,7 +275,7 @@ bool FeatureParams::FormatFullAddress(m2::PointD const & pt, string & res) const
string FeatureParams::GetStreet() const
{
return m_addrTags.Get(AddressData::FAD_STREET);
return m_addrTags.Get(AddressData::STREET);
}
void FeatureParams::SetGeomType(feature::EGeomType t)

3
indexer/feature_data.hpp
View file

@ -274,6 +274,9 @@ public:
feature::Metadata const & GetMetadata() const { return m_metadata; }
feature::Metadata & GetMetadata() { return m_metadata; }
/// @param[in] fullStoring \n
/// - true when saving in temporary files after first generation step \n
/// - false when final mwm saving
template <class SinkT> void Write(SinkT & sink, bool fullStoring) const
{
uint8_t const header = GetHeader();

6
indexer/feature_loader.cpp
View file

@ -278,9 +278,9 @@ void LoaderCurrent::ParseMetadata()
if (it != idx.end() && m_pF->m_id.m_index == it->key)
{
ReaderSource<FilesContainerR::ReaderT> reader(m_Info.GetMetadataReader());
reader.Skip(it->value);
m_pF->GetMetadata().DeserializeFromMWM(reader);
ReaderSource<FilesContainerR::ReaderT> src(m_Info.GetMetadataReader());
src.Skip(it->value);
m_pF->GetMetadata().DeserializeFromMWM(src);
}
}
catch (Reader::OpenException const &)

77
indexer/feature_meta.hpp
View file

@ -32,34 +32,29 @@ namespace feature
return types;
}
void Drop(uint8_t type)
{
m_metadata.erase(type);
}
inline bool Empty() const { return m_metadata.empty(); }
inline size_t Size() const { return m_metadata.size(); }
template <class ArchiveT> void Serialize(ArchiveT & ar) const
template <class TSink> void Serialize(TSink & sink) const
{
uint8_t const sz = m_metadata.size();
WriteToSink(ar, sz);
WriteToSink(sink, sz);
for (auto const & it : m_metadata)
{
WriteToSink(ar, static_cast<uint8_t>(it.first));
utils::WriteString(ar, it.second);
WriteToSink(sink, static_cast<uint8_t>(it.first));
utils::WriteString(sink, it.second);
}
}
template <class ArchiveT> void Deserialize(ArchiveT & ar)
template <class TSource> void Deserialize(TSource & src)
{
uint8_t const sz = ReadPrimitiveFromSource<uint8_t>(ar);
uint8_t const sz = ReadPrimitiveFromSource<uint8_t>(src);
for (size_t i = 0; i < sz; ++i)
{
uint8_t const key = ReadPrimitiveFromSource<uint8_t>(ar);
uint8_t const key = ReadPrimitiveFromSource<uint8_t>(src);
string value;
utils::ReadString(ar, value);
m_metadata.insert(make_pair(key, value));
utils::ReadString(src, value);
m_metadata[key].swap(value);
}
}
@ -100,7 +95,7 @@ namespace feature
static_assert(FMD_COUNT <= 255, "Meta types count is limited to one byte.");
void Add(EType type, string const & s)
void Set(EType type, string const & value)
{
auto found = m_metadata.find(type);
if (found == m_metadata.end())
@ -117,40 +112,11 @@ namespace feature
}
}
/// Synonym of Set(type, "").
void Drop(EType type)
{
Set(type, "");
}
string Get(EType type) const
{
auto const it = m_metadata.find(type);
return (it == m_metadata.end()) ? string() : it->second;
}
vector<EType> GetPresentTypes() const
{
vector<EType> types;
types.reserve(m_metadata.size());
for (auto const & item : m_metadata)
types.push_back(item.first);
return types;
}
inline bool Empty() const { return m_metadata.empty(); }
inline size_t Size() const { return m_metadata.size(); }
void Drop(EType type) { Set(type, ""); }
string GetWikiURL() const;
=======
val = val + ", " + s;
}
>>>>>>> 12268b5... [generator] Pass address tokens to the search index generation step.
template <class ArchiveT> void SerializeToMWM(ArchiveT & ar) const
template <class TWriter> void SerializeToMWM(TWriter & writer) const
{
for (auto const & e : m_metadata)
{
@ -158,19 +124,19 @@ namespace feature
uint8_t const mark = (&e == &(*m_metadata.crbegin()) ? 0x80 : 0);
uint8_t elem[2] = {static_cast<uint8_t>(e.first | mark),
static_cast<uint8_t>(min(e.second.size(), (size_t)kMaxStringLength))};
ar.Write(elem, sizeof(elem));
ar.Write(e.second.data(), elem[1]);
writer.Write(elem, sizeof(elem));
writer.Write(e.second.data(), elem[1]);
}
}
template <class ArchiveT> void DeserializeFromMWM(ArchiveT & ar)
template <class TSource> void DeserializeFromMWM(TSource & src)
{
uint8_t header[2] = {0};
char buffer[kMaxStringLength] = {0};
do
{
ar.Read(header, sizeof(header));
ar.Read(buffer, header[1]);
src.Read(header, sizeof(header));
src.Read(buffer, header[1]);
m_metadata[header[0] & 0x7F].assign(buffer, header[1]);
} while (!(header[0] & 0x80));
}
@ -182,14 +148,9 @@ namespace feature
class AddressData : public MetadataBase
{
public:
enum EType
{
FAD_PLACE = 1,
FAD_STREET = 2,
FAD_POSTCODE = 3,
};
enum Type { PLACE, STREET, POSTCODE };
void Add(EType type, string const & s)
void Add(Type type, string const & s)
{
/// @todo Probably, we need to add separator here and store multiple values.
m_metadata[type] = s;

2
indexer/search_index_builder.cpp
View file

@ -286,7 +286,7 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest & features, CategoriesHolder &
inserter.m_val.m_featureId = index++;
string const street = data.Get(feature::AddressData::FAD_STREET);
string const street = data.Get(feature::AddressData::STREET);
if (!street.empty())
inserter(lang, street);
}