forked from organicmaps/organicmaps
[generator] Separated readers and writers in IntermediateData.
This commit is contained in:
Maxim Pimenov
Vladimir Byko-Ianko
parent
89df1cc319
commit
053bbb0a59
8 changed files with 644 additions and 537 deletions
|
|
@ -39,6 +39,7 @@ set(SRC
|
|||
gen_mwm_info.hpp
|
||||
generate_info.hpp
|
||||
geometry_holder.hpp
|
||||
intermediate_data.cpp
|
||||
intermediate_data.hpp
|
||||
intermediate_elements.hpp
|
||||
locality_sorter.cpp
|
||||
|
|
|
|||
|
|
@ -1,12 +1,19 @@
|
|||
#include "testing/testing.hpp"
|
||||
|
||||
#include "coding/parse_xml.hpp"
|
||||
#include "generator/osm_source.hpp"
|
||||
#include "generator/generator_tests/source_data.hpp"
|
||||
#include "generator/osm_element.hpp"
|
||||
#include "generator/osm_source.hpp"
|
||||
|
||||
#include "coding/parse_xml.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "source_data.hpp"
|
||||
using namespace generator;
|
||||
|
||||
UNIT_TEST(Source_To_Element_create_from_xml_test)
|
||||
{
|
||||
|
|
|
|||
250
generator/intermediate_data.cpp
Normal file
250
generator/intermediate_data.cpp
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
#include "generator/intermediate_data.hpp"
|
||||
|
||||
#include <exception>
|
||||
#include <string>
|
||||
|
||||
#include "base/logging.hpp"
|
||||
|
||||
#include "defines.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace
|
||||
{
|
||||
size_t const kFlushCount = 1024;
|
||||
double const kValueOrder = 1e7;
|
||||
|
||||
void ToLatLon(double lat, double lon, generator::cache::LatLon & ll)
|
||||
{
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lon64 = lon * kValueOrder;
|
||||
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
ll.lon = static_cast<int32_t>(lon64);
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lon64, ("Longtitude is out of 32bit boundary!"));
|
||||
}
|
||||
|
||||
bool FromLatLon(generator::cache::LatLon const & ll, double & lat, double & lon)
|
||||
{
|
||||
// Assume that a valid coordinate is not (0, 0).
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
{
|
||||
lat = static_cast<double>(ll.lat) / kValueOrder;
|
||||
lon = static_cast<double>(ll.lon) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
lat = 0.0;
|
||||
lon = 0.0;
|
||||
return false;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace generator
|
||||
{
|
||||
namespace cache
|
||||
{
|
||||
// IndexFileReader ---------------------------------------------------------------------------------
|
||||
IndexFileReader::IndexFileReader(string const & name) : m_fileReader(name.c_str()) {}
|
||||
|
||||
void IndexFileReader::ReadAll()
|
||||
{
|
||||
m_elements.clear();
|
||||
size_t fileSize = m_fileReader.Size();
|
||||
if (fileSize == 0)
|
||||
return;
|
||||
|
||||
LOG_SHORT(LINFO, ("Offsets reading is started for file", m_fileReader.GetName()));
|
||||
CHECK_EQUAL(0, fileSize % sizeof(Element), ("Damaged file."));
|
||||
|
||||
try
|
||||
{
|
||||
m_elements.resize(base::checked_cast<size_t>(fileSize / sizeof(Element)));
|
||||
}
|
||||
catch (exception const &) // bad_alloc
|
||||
{
|
||||
LOG(LCRITICAL, ("Insufficient memory for required offset map"));
|
||||
}
|
||||
|
||||
m_fileReader.Read(0, &m_elements[0], base::checked_cast<size_t>(fileSize));
|
||||
|
||||
sort(m_elements.begin(), m_elements.end(), ElementComparator());
|
||||
|
||||
LOG_SHORT(LINFO, ("Offsets reading is finished"));
|
||||
}
|
||||
|
||||
bool IndexFileReader::GetValueByKey(Key key, Value & value) const
|
||||
{
|
||||
auto it = lower_bound(m_elements.begin(), m_elements.end(), key, ElementComparator());
|
||||
if (it != m_elements.end() && it->first == key)
|
||||
{
|
||||
value = it->second;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// IndexFileWriter ---------------------------------------------------------------------------------
|
||||
IndexFileWriter::IndexFileWriter(string const & name) : m_fileWriter(name.c_str()) {}
|
||||
|
||||
void IndexFileWriter::WriteAll()
|
||||
{
|
||||
if (m_elements.empty())
|
||||
return;
|
||||
|
||||
m_fileWriter.Write(&m_elements[0], m_elements.size() * sizeof(Element));
|
||||
m_elements.clear();
|
||||
}
|
||||
|
||||
void IndexFileWriter::Add(Key k, Value const & v)
|
||||
{
|
||||
if (m_elements.size() > kFlushCount)
|
||||
WriteAll();
|
||||
|
||||
m_elements.emplace_back(k, v);
|
||||
}
|
||||
|
||||
// OSMElementCacheReader ---------------------------------------------------------------------------
|
||||
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload)
|
||||
: m_fileReader(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload)
|
||||
{
|
||||
if (!m_preload)
|
||||
return;
|
||||
size_t sz = m_fileReader.Size();
|
||||
m_data.resize(sz);
|
||||
m_fileReader.Read(0, m_data.data(), sz);
|
||||
}
|
||||
|
||||
void OSMElementCacheReader::LoadOffsets() { m_offsets.ReadAll(); }
|
||||
|
||||
// OSMElementCacheWriter ---------------------------------------------------------------------------
|
||||
OSMElementCacheWriter::OSMElementCacheWriter(string const & name, bool preload)
|
||||
: m_fileWriter(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload)
|
||||
{
|
||||
}
|
||||
|
||||
void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); }
|
||||
|
||||
// RawFilePointStorageMmapReader -------------------------------------------------------------------
|
||||
RawFilePointStorageMmapReader::RawFilePointStorageMmapReader(string const & name)
|
||||
: m_mmapReader(name)
|
||||
{
|
||||
}
|
||||
|
||||
bool RawFilePointStorageMmapReader::GetPoint(uint64_t id, double & lat, double & lon) const
|
||||
{
|
||||
LatLon ll;
|
||||
m_mmapReader.Read(id * sizeof(ll), &ll, sizeof(ll));
|
||||
|
||||
bool ret = FromLatLon(ll, lat, lon);
|
||||
if (!ret)
|
||||
LOG(LERROR, ("Node with id =", id, " not found!"));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// RawFilePointStorageWriter -----------------------------------------------------------------------
|
||||
RawFilePointStorageWriter::RawFilePointStorageWriter(string const & name) : m_fileWriter(name) {}
|
||||
|
||||
void RawFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon)
|
||||
{
|
||||
LatLon ll;
|
||||
ToLatLon(lat, lon, ll);
|
||||
|
||||
m_fileWriter.Seek(id * sizeof(ll));
|
||||
m_fileWriter.Write(&ll, sizeof(ll));
|
||||
|
||||
++m_numProcessedPoints;
|
||||
}
|
||||
|
||||
// RawMemPointStorageReader ------------------------------------------------------------------------
|
||||
RawMemPointStorageReader::RawMemPointStorageReader(string const & name)
|
||||
: m_fileReader(name), m_data(static_cast<size_t>(1) << 33)
|
||||
{
|
||||
m_fileReader.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
|
||||
}
|
||||
|
||||
bool RawMemPointStorageReader::GetPoint(uint64_t id, double & lat, double & lon) const
|
||||
{
|
||||
LatLon const & ll = m_data[id];
|
||||
bool ret = FromLatLon(ll, lat, lon);
|
||||
if (!ret)
|
||||
LOG(LERROR, ("Node with id =", id, " not found!"));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// RawMemPointStorageWriter ------------------------------------------------------------------------
|
||||
RawMemPointStorageWriter::RawMemPointStorageWriter(string const & name)
|
||||
: m_fileWriter(name), m_data(static_cast<size_t>(1) << 33)
|
||||
{
|
||||
}
|
||||
|
||||
RawMemPointStorageWriter::~RawMemPointStorageWriter()
|
||||
{
|
||||
m_fileWriter.Write(m_data.data(), m_data.size() * sizeof(LatLon));
|
||||
}
|
||||
|
||||
void RawMemPointStorageWriter::AddPoint(uint64_t id, double lat, double lon)
|
||||
{
|
||||
CHECK_LESS(id, m_data.size(),
|
||||
("Found node with id", id, "which is bigger than the allocated cache size"));
|
||||
|
||||
LatLon & ll = m_data[id];
|
||||
ToLatLon(lat, lon, ll);
|
||||
|
||||
++m_numProcessedPoints;
|
||||
}
|
||||
|
||||
// MapFilePointStorageReader -----------------------------------------------------------------------
|
||||
MapFilePointStorageReader::MapFilePointStorageReader(string const & name)
|
||||
: m_fileReader(name + ".short")
|
||||
{
|
||||
LOG(LINFO, ("Nodes reading is started"));
|
||||
|
||||
uint64_t const count = m_fileReader.Size();
|
||||
|
||||
uint64_t pos = 0;
|
||||
while (pos < count)
|
||||
{
|
||||
LatLonPos ll;
|
||||
m_fileReader.Read(pos, &ll, sizeof(ll));
|
||||
|
||||
m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
|
||||
|
||||
pos += sizeof(ll);
|
||||
}
|
||||
|
||||
LOG(LINFO, ("Nodes reading is finished"));
|
||||
}
|
||||
|
||||
bool MapFilePointStorageReader::GetPoint(uint64_t id, double & lat, double & lon) const
|
||||
{
|
||||
auto i = m_map.find(id);
|
||||
if (i == m_map.end())
|
||||
return false;
|
||||
lat = static_cast<double>(i->second.first) / kValueOrder;
|
||||
lon = static_cast<double>(i->second.second) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
|
||||
// MapFilePointStorageWriter -----------------------------------------------------------------------
|
||||
MapFilePointStorageWriter::MapFilePointStorageWriter(string const & name)
|
||||
: m_fileWriter(name + ".short")
|
||||
{
|
||||
}
|
||||
|
||||
void MapFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon)
|
||||
{
|
||||
LatLon ll;
|
||||
ToLatLon(lat, lon, ll);
|
||||
|
||||
LatLonPos llp;
|
||||
llp.pos = id;
|
||||
llp.lat = ll.lat;
|
||||
llp.lon = ll.lon;
|
||||
|
||||
m_fileWriter.Write(&llp, sizeof(llp));
|
||||
|
||||
++m_numProcessedPoints;
|
||||
}
|
||||
} // namespace cache
|
||||
} // namespace generator
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include "generator/generate_info.hpp"
|
||||
#include "generator/intermediate_elements.hpp"
|
||||
|
||||
#include "coding/file_name_utils.hpp"
|
||||
|
|
@ -10,9 +11,8 @@
|
|||
#include "base/logging.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <exception>
|
||||
#include <fstream>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
|
|
@ -21,105 +21,45 @@
|
|||
|
||||
#include "defines.hpp"
|
||||
|
||||
/// Classes for reading and writing any data in file with map of offsets for
|
||||
/// fast searching in memory by some key.
|
||||
// Classes for reading and writing any data in file with map of offsets for
|
||||
// fast searching in memory by some key.
|
||||
namespace generator
|
||||
{
|
||||
namespace cache
|
||||
{
|
||||
using Key = uint64_t;
|
||||
|
||||
enum class EMode { Write = true, Read = false };
|
||||
|
||||
namespace detail
|
||||
// Used to store all world nodes inside temporary index file.
|
||||
// To find node by id, just calculate offset inside index file:
|
||||
// offset_in_file = sizeof(LatLon) * node_ID
|
||||
struct LatLon
|
||||
{
|
||||
template <class TFile, class TValue>
|
||||
class IndexFile
|
||||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
static_assert(sizeof(LatLon) == 8, "Invalid structure size");
|
||||
|
||||
struct LatLonPos
|
||||
{
|
||||
using TKey = uint64_t;
|
||||
static_assert(is_integral<TKey>::value, "TKey is not integral type");
|
||||
using TElement = std::pair<TKey, TValue>;
|
||||
using TContainer = std::vector<TElement>;
|
||||
|
||||
TContainer m_elements;
|
||||
TFile m_file;
|
||||
|
||||
static size_t constexpr kFlushCount = 1024;
|
||||
|
||||
struct ElementComparator
|
||||
{
|
||||
bool operator()(TElement const & r1, TElement const & r2) const
|
||||
{
|
||||
return ((r1.first == r2.first) ? r1.second < r2.second : r1.first < r2.first);
|
||||
}
|
||||
bool operator()(TElement const & r1, TKey r2) const { return (r1.first < r2); }
|
||||
bool operator()(TKey r1, TElement const & r2) const { return (r1 < r2.first); }
|
||||
};
|
||||
|
||||
static size_t CheckedCast(uint64_t v)
|
||||
{
|
||||
ASSERT_LESS(v, std::numeric_limits<size_t>::max(), ("Value too long for memory address : ", v));
|
||||
return static_cast<size_t>(v);
|
||||
}
|
||||
uint64_t pos;
|
||||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
|
||||
|
||||
class IndexFileReader
|
||||
{
|
||||
public:
|
||||
explicit IndexFile(std::string const & name) : m_file(name.c_str()) {}
|
||||
using Value = uint64_t;
|
||||
|
||||
std::string GetFileName() const { return m_file.GetName(); }
|
||||
explicit IndexFileReader(std::string const & name);
|
||||
|
||||
void WriteAll()
|
||||
{
|
||||
if (m_elements.empty())
|
||||
return;
|
||||
void ReadAll();
|
||||
|
||||
m_file.Write(&m_elements[0], m_elements.size() * sizeof(TElement));
|
||||
m_elements.clear();
|
||||
}
|
||||
bool GetValueByKey(Key key, Value & value) const;
|
||||
|
||||
void ReadAll()
|
||||
{
|
||||
m_elements.clear();
|
||||
size_t fileSize = m_file.Size();
|
||||
if (fileSize == 0)
|
||||
return;
|
||||
|
||||
LOG_SHORT(LINFO, ("Offsets reading is started for file ", GetFileName()));
|
||||
CHECK_EQUAL(0, fileSize % sizeof(TElement), ("Damaged file."));
|
||||
|
||||
try
|
||||
{
|
||||
m_elements.resize(CheckedCast(fileSize / sizeof(TElement)));
|
||||
}
|
||||
catch (std::exception const &) // bad_alloc
|
||||
{
|
||||
LOG(LCRITICAL, ("Insufficient memory for required offset map"));
|
||||
}
|
||||
|
||||
m_file.Read(0, &m_elements[0], CheckedCast(fileSize));
|
||||
|
||||
std::sort(m_elements.begin(), m_elements.end(), ElementComparator());
|
||||
|
||||
LOG_SHORT(LINFO, ("Offsets reading is finished"));
|
||||
}
|
||||
|
||||
void Add(TKey k, TValue const & v)
|
||||
{
|
||||
if (m_elements.size() > kFlushCount)
|
||||
WriteAll();
|
||||
|
||||
m_elements.push_back(std::make_pair(k, v));
|
||||
}
|
||||
|
||||
bool GetValueByKey(TKey key, TValue & value) const
|
||||
{
|
||||
auto it = std::lower_bound(m_elements.begin(), m_elements.end(), key, ElementComparator());
|
||||
if ((it != m_elements.end()) && ((*it).first == key))
|
||||
{
|
||||
value = (*it).second;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachByKey(TKey k, ToDo && toDo) const
|
||||
template <typename ToDo>
|
||||
void ForEachByKey(Key k, ToDo && toDo) const
|
||||
{
|
||||
auto range = std::equal_range(m_elements.begin(), m_elements.end(), k, ElementComparator());
|
||||
for (; range.first != range.second; ++range.first)
|
||||
|
|
@ -128,73 +68,55 @@ public:
|
|||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
template <EMode TMode>
|
||||
class OSMElementCache
|
||||
private:
|
||||
static_assert(is_integral<Key>::value, "Key must be an integral type");
|
||||
using Element = std::pair<Key, Value>;
|
||||
|
||||
struct ElementComparator
|
||||
{
|
||||
bool operator()(Element const & r1, Element const & r2) const
|
||||
{
|
||||
return ((r1.first == r2.first) ? r1.second < r2.second : r1.first < r2.first);
|
||||
}
|
||||
bool operator()(Element const & r1, Key r2) const { return (r1.first < r2); }
|
||||
bool operator()(Key r1, Element const & r2) const { return (r1 < r2.first); }
|
||||
};
|
||||
|
||||
std::vector<Element> m_elements;
|
||||
FileReader m_fileReader;
|
||||
};
|
||||
|
||||
class IndexFileWriter
|
||||
{
|
||||
public:
|
||||
using TKey = uint64_t;
|
||||
using TStorage = std::conditional_t<TMode == EMode::Write, FileWriter, FileReader>;
|
||||
using TOffsetFile = std::conditional_t<TMode == EMode::Write, FileWriter, FileReader>;
|
||||
using Value = uint64_t;
|
||||
|
||||
protected:
|
||||
using TBuffer = std::vector<uint8_t>;
|
||||
TStorage m_storage;
|
||||
detail::IndexFile<TOffsetFile, uint64_t> m_offsets;
|
||||
std::string m_name;
|
||||
TBuffer m_data;
|
||||
bool m_preload = false;
|
||||
explicit IndexFileWriter(std::string const & name);
|
||||
|
||||
void WriteAll();
|
||||
|
||||
void Add(Key k, Value const & v);
|
||||
|
||||
private:
|
||||
using Element = std::pair<Key, Value>;
|
||||
|
||||
std::vector<Element> m_elements;
|
||||
FileWriter m_fileWriter;
|
||||
};
|
||||
|
||||
class OSMElementCacheReader
|
||||
{
|
||||
public:
|
||||
OSMElementCache(std::string const & name, bool preload = false)
|
||||
: m_storage(name)
|
||||
, m_offsets(name + OFFSET_EXT)
|
||||
, m_name(name)
|
||||
, m_preload(preload)
|
||||
{
|
||||
InitStorage<TMode>();
|
||||
}
|
||||
OSMElementCacheReader(std::string const & name, bool preload = false);
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Write, void> InitStorage()
|
||||
{
|
||||
}
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Read, void> InitStorage()
|
||||
{
|
||||
if (!m_preload)
|
||||
return;
|
||||
size_t sz = m_storage.Size();
|
||||
m_data.resize(sz);
|
||||
m_storage.Read(0, m_data.data(), sz);
|
||||
}
|
||||
|
||||
template <class TValue, EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Write, void> Write(TKey id, TValue const & value)
|
||||
{
|
||||
m_offsets.Add(id, m_storage.Pos());
|
||||
m_data.clear();
|
||||
MemWriter<TBuffer> w(m_data);
|
||||
|
||||
value.Write(w);
|
||||
|
||||
// write buffer
|
||||
ASSERT_LESS(m_data.size(), std::numeric_limits<uint32_t>::max(), ());
|
||||
uint32_t sz = static_cast<uint32_t>(m_data.size());
|
||||
m_storage.Write(&sz, sizeof(sz));
|
||||
m_storage.Write(m_data.data(), sz * sizeof(TBuffer::value_type));
|
||||
}
|
||||
|
||||
template <class TValue, EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Read, bool> Read(TKey id, TValue & value)
|
||||
template <class Value>
|
||||
bool Read(Key id, Value & value)
|
||||
{
|
||||
uint64_t pos = 0;
|
||||
if (!m_offsets.GetValueByKey(id, pos))
|
||||
{
|
||||
LOG_SHORT(LWARNING, ("Can't find offset in file", m_offsets.GetFileName(), "by id", id));
|
||||
LOG_SHORT(LWARNING, ("Can't find offset in file", m_name + OFFSET_EXT, "by id", id));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -204,9 +126,9 @@ public:
|
|||
if (!m_preload)
|
||||
{
|
||||
// in case not-in-memory work we read buffer
|
||||
m_storage.Read(pos, &valueSize, sizeof(valueSize));
|
||||
m_fileReader.Read(pos, &valueSize, sizeof(valueSize));
|
||||
m_data.resize(valueSize);
|
||||
m_storage.Read(pos + sizeof(valueSize), m_data.data(), valueSize);
|
||||
m_fileReader.Read(pos + sizeof(valueSize), m_data.data(), valueSize);
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
|
|
@ -215,222 +137,273 @@ public:
|
|||
return true;
|
||||
}
|
||||
|
||||
inline void SaveOffsets() { m_offsets.WriteAll(); }
|
||||
inline void LoadOffsets() { m_offsets.ReadAll(); }
|
||||
void LoadOffsets();
|
||||
|
||||
protected:
|
||||
FileReader m_fileReader;
|
||||
IndexFileReader m_offsets;
|
||||
std::string m_name;
|
||||
std::vector<uint8_t> m_data;
|
||||
bool m_preload = false;
|
||||
};
|
||||
|
||||
/// Used to store all world nodes inside temporary index file.
|
||||
/// To find node by id, just calculate offset inside index file:
|
||||
/// offset_in_file = sizeof(LatLon) * node_ID
|
||||
class PointStorage
|
||||
class OSMElementCacheWriter
|
||||
{
|
||||
size_t m_processedPoint = 0;
|
||||
|
||||
public:
|
||||
struct LatLon
|
||||
OSMElementCacheWriter(std::string const & name, bool preload = false);
|
||||
|
||||
template <typename Value>
|
||||
void Write(Key id, Value const & value)
|
||||
{
|
||||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
static_assert(sizeof(LatLon) == 8, "Invalid structure size");
|
||||
m_offsets.Add(id, m_fileWriter.Pos());
|
||||
m_data.clear();
|
||||
MemWriter<decltype(m_data)> w(m_data);
|
||||
|
||||
struct LatLonPos
|
||||
{
|
||||
uint64_t pos;
|
||||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
|
||||
value.Write(w);
|
||||
|
||||
inline size_t GetProcessedPoint() const { return m_processedPoint; }
|
||||
inline void IncProcessedPoint() { ++m_processedPoint; }
|
||||
};
|
||||
|
||||
template <EMode TMode>
|
||||
class RawFilePointStorage : public PointStorage
|
||||
{
|
||||
#ifdef OMIM_OS_WINDOWS
|
||||
using TFileReader = FileReader;
|
||||
#else
|
||||
using TFileReader = MmapReader;
|
||||
#endif
|
||||
|
||||
std::conditional_t<TMode == EMode::Write, FileWriter, TFileReader> m_file;
|
||||
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
|
||||
public:
|
||||
explicit RawFilePointStorage(std::string const & name) : m_file(name) {}
|
||||
|
||||
template <EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Write, void> AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
LatLon ll;
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
ll.lon = static_cast<int32_t>(lng64);
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
|
||||
|
||||
m_file.Seek(id * sizeof(ll));
|
||||
m_file.Write(&ll, sizeof(ll));
|
||||
|
||||
IncProcessedPoint();
|
||||
ASSERT_LESS(m_data.size(), std::numeric_limits<uint32_t>::max(), ());
|
||||
uint32_t sz = static_cast<uint32_t>(m_data.size());
|
||||
// ?!
|
||||
m_fileWriter.Write(&sz, sizeof(sz));
|
||||
m_fileWriter.Write(m_data.data(), sz);
|
||||
}
|
||||
|
||||
template <EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Read, bool> GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
LatLon ll;
|
||||
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
|
||||
void SaveOffsets();
|
||||
|
||||
// assume that valid coordinate is not (0, 0)
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
{
|
||||
lat = static_cast<double>(ll.lat) / kValueOrder;
|
||||
lng = static_cast<double>(ll.lon) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
LOG(LERROR, ("Node with id = ", id, " not found!"));
|
||||
return false;
|
||||
}
|
||||
protected:
|
||||
FileWriter m_fileWriter;
|
||||
IndexFileWriter m_offsets;
|
||||
std::string m_name;
|
||||
std::vector<uint8_t> m_data;
|
||||
bool m_preload = false;
|
||||
};
|
||||
|
||||
template <EMode TMode>
|
||||
class RawMemPointStorage : public PointStorage
|
||||
class RawFilePointStorageMmapReader
|
||||
{
|
||||
std::conditional_t<TMode == EMode::Write, FileWriter, FileReader> m_file;
|
||||
public:
|
||||
explicit RawFilePointStorageMmapReader(std::string const & name);
|
||||
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
bool GetPoint(uint64_t id, double & lat, double & lon) const;
|
||||
|
||||
private:
|
||||
MmapReader m_mmapReader;
|
||||
};
|
||||
|
||||
class RawFilePointStorageWriter
|
||||
{
|
||||
public:
|
||||
explicit RawFilePointStorageWriter(std::string const & name);
|
||||
|
||||
void AddPoint(uint64_t id, double lat, double lon);
|
||||
uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; }
|
||||
|
||||
private:
|
||||
FileWriter m_fileWriter;
|
||||
uint64_t m_numProcessedPoints = 0;
|
||||
};
|
||||
|
||||
class RawMemPointStorageReader
|
||||
{
|
||||
public:
|
||||
explicit RawMemPointStorageReader(std::string const & name);
|
||||
|
||||
bool GetPoint(uint64_t id, double & lat, double & lon) const;
|
||||
|
||||
private:
|
||||
FileReader m_fileReader;
|
||||
std::vector<LatLon> m_data;
|
||||
|
||||
public:
|
||||
explicit RawMemPointStorage(std::string const & name) : m_file(name), m_data(static_cast<size_t>(1) << 33)
|
||||
{
|
||||
InitStorage<TMode>();
|
||||
}
|
||||
|
||||
~RawMemPointStorage() { DoneStorage<TMode>(); }
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Write, void> InitStorage()
|
||||
{
|
||||
}
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Read, void> InitStorage()
|
||||
{
|
||||
m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
|
||||
}
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Write, void> DoneStorage()
|
||||
{
|
||||
m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon));
|
||||
}
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Read, void> DoneStorage()
|
||||
{
|
||||
}
|
||||
|
||||
template <EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Write, void> AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
CHECK_LESS(id, m_data.size(), ("Found node with id", id, "which is bigger than the allocated cache size"));
|
||||
LatLon & ll = m_data[id];
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
ll.lon = static_cast<int32_t>(lng64);
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
|
||||
|
||||
IncProcessedPoint();
|
||||
}
|
||||
|
||||
template <EMode T = TMode>
|
||||
std::enable_if_t<T == EMode::Read, bool> GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
LatLon const & ll = m_data[id];
|
||||
// assume that valid coordinate is not (0, 0)
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
{
|
||||
lat = static_cast<double>(ll.lat) / kValueOrder;
|
||||
lng = static_cast<double>(ll.lon) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
LOG(LERROR, ("Node with id = ", id, " not found!"));
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <EMode TMode>
|
||||
class MapFilePointStorage : public PointStorage
|
||||
class RawMemPointStorageWriter
|
||||
{
|
||||
std::conditional_t<TMode == EMode::Write, FileWriter, FileReader> m_file;
|
||||
std::unordered_map<uint64_t, std::pair<int32_t, int32_t>> m_map;
|
||||
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
|
||||
public:
|
||||
explicit MapFilePointStorage(std::string const & name) : m_file(name + ".short") { InitStorage<TMode>(); }
|
||||
explicit RawMemPointStorageWriter(std::string const & name);
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Write, void> InitStorage()
|
||||
{
|
||||
}
|
||||
~RawMemPointStorageWriter();
|
||||
|
||||
template <EMode T>
|
||||
std::enable_if_t<T == EMode::Read, void> InitStorage()
|
||||
{
|
||||
LOG(LINFO, ("Nodes reading is started"));
|
||||
void AddPoint(uint64_t id, double lat, double lon);
|
||||
uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; }
|
||||
|
||||
uint64_t const count = m_file.Size();
|
||||
|
||||
uint64_t pos = 0;
|
||||
while (pos < count)
|
||||
{
|
||||
LatLonPos ll;
|
||||
m_file.Read(pos, &ll, sizeof(ll));
|
||||
|
||||
m_map.emplace(std::make_pair(ll.pos, std::make_pair(ll.lat, ll.lon)));
|
||||
|
||||
pos += sizeof(ll);
|
||||
}
|
||||
|
||||
LOG(LINFO, ("Nodes reading is finished"));
|
||||
}
|
||||
|
||||
void AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
LatLonPos ll;
|
||||
ll.pos = id;
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
ll.lon = static_cast<int32_t>(lng64);
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
|
||||
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
|
||||
m_file.Write(&ll, sizeof(ll));
|
||||
|
||||
IncProcessedPoint();
|
||||
}
|
||||
|
||||
bool GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
auto i = m_map.find(id);
|
||||
if (i == m_map.end())
|
||||
return false;
|
||||
lat = static_cast<double>(i->second.first) / kValueOrder;
|
||||
lng = static_cast<double>(i->second.second) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
private:
|
||||
FileWriter m_fileWriter;
|
||||
std::vector<LatLon> m_data;
|
||||
uint64_t m_numProcessedPoints = 0;
|
||||
};
|
||||
|
||||
class MapFilePointStorageReader
|
||||
{
|
||||
public:
|
||||
explicit MapFilePointStorageReader(std::string const & name);
|
||||
|
||||
bool GetPoint(uint64_t id, double & lat, double & lon) const;
|
||||
|
||||
private:
|
||||
FileReader m_fileReader;
|
||||
std::unordered_map<uint64_t, std::pair<int32_t, int32_t>> m_map;
|
||||
};
|
||||
|
||||
class MapFilePointStorageWriter
|
||||
{
|
||||
public:
|
||||
explicit MapFilePointStorageWriter(std::string const & name);
|
||||
|
||||
void AddPoint(uint64_t id, double lat, double lon);
|
||||
uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; }
|
||||
|
||||
private:
|
||||
FileWriter m_fileWriter;
|
||||
std::unordered_map<uint64_t, std::pair<int32_t, int32_t>> m_map;
|
||||
uint64_t m_numProcessedPoints = 0;
|
||||
};
|
||||
|
||||
template <typename NodesHolder>
|
||||
class IntermediateDataReader
|
||||
{
|
||||
public:
|
||||
IntermediateDataReader(NodesHolder & nodes, feature::GenerateInfo & info)
|
||||
: m_nodes(nodes)
|
||||
, m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache)
|
||||
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache)
|
||||
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
|
||||
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))
|
||||
{
|
||||
}
|
||||
|
||||
bool GetNode(Key id, double & lat, double & lon) { return m_nodes.GetPoint(id, lat, lon); }
|
||||
bool GetWay(Key id, WayElement & e) { return m_ways.Read(id, e); }
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByWay(Key id, ToDo && toDo)
|
||||
{
|
||||
RelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_wayToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByWayCached(Key id, ToDo && toDo)
|
||||
{
|
||||
CachedRelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_wayToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByNodeCached(Key id, ToDo && toDo)
|
||||
{
|
||||
CachedRelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_nodeToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
void LoadIndex()
|
||||
{
|
||||
m_ways.LoadOffsets();
|
||||
m_relations.LoadOffsets();
|
||||
|
||||
m_nodeToRelations.ReadAll();
|
||||
m_wayToRelations.ReadAll();
|
||||
}
|
||||
|
||||
private:
|
||||
using CacheReader = cache::OSMElementCacheReader;
|
||||
|
||||
template <class Element, class ToDo>
|
||||
struct ElementProcessorBase
|
||||
{
|
||||
protected:
|
||||
CacheReader m_reader;
|
||||
ToDo & m_toDo;
|
||||
|
||||
public:
|
||||
ElementProcessorBase(CacheReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
|
||||
|
||||
bool operator()(uint64_t id)
|
||||
{
|
||||
Element e;
|
||||
return m_reader.Read(id, e) ? m_toDo(id, e) : false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class ToDo>
|
||||
struct RelationProcessor : public ElementProcessorBase<RelationElement, ToDo>
|
||||
{
|
||||
using Base = ElementProcessorBase<RelationElement, ToDo>;
|
||||
|
||||
RelationProcessor(CacheReader & reader, ToDo & toDo) : Base(reader, toDo) {}
|
||||
};
|
||||
|
||||
template <class ToDo>
|
||||
struct CachedRelationProcessor : public RelationProcessor<ToDo>
|
||||
{
|
||||
using Base = RelationProcessor<ToDo>;
|
||||
|
||||
CachedRelationProcessor(CacheReader & reader, ToDo & toDo) : Base(reader, toDo) {}
|
||||
bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); }
|
||||
};
|
||||
|
||||
NodesHolder & m_nodes;
|
||||
|
||||
cache::OSMElementCacheReader m_ways;
|
||||
cache::OSMElementCacheReader m_relations;
|
||||
|
||||
cache::IndexFileReader m_nodeToRelations;
|
||||
cache::IndexFileReader m_wayToRelations;
|
||||
};
|
||||
|
||||
template <typename NodesHolder>
|
||||
class IntermediateDataWriter
|
||||
{
|
||||
public:
|
||||
IntermediateDataWriter(NodesHolder & nodes, feature::GenerateInfo & info)
|
||||
: m_nodes(nodes)
|
||||
, m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache)
|
||||
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache)
|
||||
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
|
||||
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))
|
||||
{
|
||||
}
|
||||
|
||||
void AddNode(Key id, double lat, double lon) { m_nodes.AddPoint(id, lat, lon); }
|
||||
|
||||
void AddWay(Key id, WayElement const & e) { m_ways.Write(id, e); }
|
||||
|
||||
void AddRelation(Key id, RelationElement const & e)
|
||||
{
|
||||
string const & relationType = e.GetType();
|
||||
if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" ||
|
||||
relationType == "associatedStreet" || relationType == "building" ||
|
||||
relationType == "restriction"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
m_relations.Write(id, e);
|
||||
AddToIndex(m_nodeToRelations, id, e.nodes);
|
||||
AddToIndex(m_wayToRelations, id, e.ways);
|
||||
}
|
||||
|
||||
void SaveIndex()
|
||||
{
|
||||
m_ways.SaveOffsets();
|
||||
m_relations.SaveOffsets();
|
||||
|
||||
m_nodeToRelations.WriteAll();
|
||||
m_wayToRelations.WriteAll();
|
||||
}
|
||||
|
||||
private:
|
||||
NodesHolder & m_nodes;
|
||||
|
||||
cache::OSMElementCacheWriter m_ways;
|
||||
cache::OSMElementCacheWriter m_relations;
|
||||
|
||||
cache::IndexFileWriter m_nodeToRelations;
|
||||
cache::IndexFileWriter m_wayToRelations;
|
||||
|
||||
template <class Index, class Container>
|
||||
static void AddToIndex(Index & index, Key relationId, Container const & values)
|
||||
{
|
||||
for (auto const & v : values)
|
||||
index.Add(v.first, relationId);
|
||||
}
|
||||
};
|
||||
} // namespace cache
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -39,165 +39,8 @@
|
|||
|
||||
using namespace std;
|
||||
|
||||
SourceReader::SourceReader()
|
||||
: m_file(unique_ptr<istream, Deleter>(&cin, Deleter(false)))
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from stdin"));
|
||||
}
|
||||
|
||||
SourceReader::SourceReader(string const & filename)
|
||||
: m_file(unique_ptr<istream, Deleter>(new ifstream(filename), Deleter()))
|
||||
{
|
||||
CHECK(static_cast<ifstream *>(m_file.get())->is_open() , ("Can't open file:", filename));
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from", filename));
|
||||
}
|
||||
|
||||
SourceReader::SourceReader(istringstream & stream)
|
||||
: m_file(unique_ptr<istream, Deleter>(&stream, Deleter(false)))
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from memory"));
|
||||
}
|
||||
|
||||
uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
m_file->read(buffer, bufferSize);
|
||||
return m_file->gcount();
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class TNodesHolder, cache::EMode TMode>
|
||||
class IntermediateData
|
||||
{
|
||||
using TReader = cache::OSMElementCache<TMode>;
|
||||
|
||||
using TFile = conditional_t<TMode == cache::EMode::Write, FileWriter, FileReader>;
|
||||
|
||||
using TKey = uint64_t;
|
||||
static_assert(is_integral<TKey>::value, "TKey is not integral type");
|
||||
|
||||
using TIndex = cache::detail::IndexFile<TFile, TKey>;
|
||||
|
||||
TNodesHolder & m_nodes;
|
||||
|
||||
TReader m_ways;
|
||||
TReader m_relations;
|
||||
|
||||
TIndex m_nodeToRelations;
|
||||
TIndex m_wayToRelations;
|
||||
|
||||
template <class TElement, class ToDo>
|
||||
struct ElementProcessorBase
|
||||
{
|
||||
protected:
|
||||
TReader & m_reader;
|
||||
ToDo & m_toDo;
|
||||
|
||||
public:
|
||||
ElementProcessorBase(TReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
|
||||
|
||||
bool operator()(uint64_t id)
|
||||
{
|
||||
TElement e;
|
||||
return m_reader.Read(id, e) ? m_toDo(id, e) : false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class ToDo>
|
||||
struct RelationProcessor : public ElementProcessorBase<RelationElement, ToDo>
|
||||
{
|
||||
using TBase = ElementProcessorBase<RelationElement, ToDo>;
|
||||
|
||||
RelationProcessor(TReader & reader, ToDo & toDo) : TBase(reader, toDo) {}
|
||||
};
|
||||
|
||||
template <class ToDo>
|
||||
struct CachedRelationProcessor : public RelationProcessor<ToDo>
|
||||
{
|
||||
using TBase = RelationProcessor<ToDo>;
|
||||
|
||||
CachedRelationProcessor(TReader & rels, ToDo & toDo) : TBase(rels, toDo) {}
|
||||
bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); }
|
||||
};
|
||||
|
||||
template <class TIndex, class TContainer>
|
||||
static void AddToIndex(TIndex & index, TKey relationId, TContainer const & values)
|
||||
{
|
||||
for (auto const & v : values)
|
||||
index.Add(v.first, relationId);
|
||||
}
|
||||
|
||||
public:
|
||||
IntermediateData(TNodesHolder & nodes, feature::GenerateInfo & info)
|
||||
: m_nodes(nodes)
|
||||
, m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache)
|
||||
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache)
|
||||
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
|
||||
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE,ID2REL_EXT))
|
||||
{
|
||||
}
|
||||
|
||||
void AddNode(TKey id, double lat, double lng) { m_nodes.AddPoint(id, lat, lng); }
|
||||
bool GetNode(TKey id, double & lat, double & lng) { return m_nodes.GetPoint(id, lat, lng); }
|
||||
|
||||
void AddWay(TKey id, WayElement const & e) { m_ways.Write(id, e); }
|
||||
bool GetWay(TKey id, WayElement & e) { return m_ways.Read(id, e); }
|
||||
|
||||
void AddRelation(TKey id, RelationElement const & e)
|
||||
{
|
||||
string const & relationType = e.GetType();
|
||||
if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" ||
|
||||
relationType == "associatedStreet" || relationType == "building" ||
|
||||
relationType == "restriction"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
m_relations.Write(id, e);
|
||||
AddToIndex(m_nodeToRelations, id, e.nodes);
|
||||
AddToIndex(m_wayToRelations, id, e.ways);
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByWay(TKey id, ToDo && toDo)
|
||||
{
|
||||
RelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_wayToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByNodeCached(TKey id, ToDo && toDo)
|
||||
{
|
||||
CachedRelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_nodeToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo>
|
||||
void ForEachRelationByWayCached(TKey id, ToDo && toDo)
|
||||
{
|
||||
CachedRelationProcessor<ToDo> processor(m_relations, toDo);
|
||||
m_wayToRelations.ForEachByKey(id, processor);
|
||||
}
|
||||
|
||||
void SaveIndex()
|
||||
{
|
||||
m_ways.SaveOffsets();
|
||||
m_relations.SaveOffsets();
|
||||
|
||||
m_nodeToRelations.WriteAll();
|
||||
m_wayToRelations.WriteAll();
|
||||
}
|
||||
|
||||
void LoadIndex()
|
||||
{
|
||||
m_ways.LoadOffsets();
|
||||
m_relations.LoadOffsets();
|
||||
|
||||
m_nodeToRelations.ReadAll();
|
||||
m_wayToRelations.ReadAll();
|
||||
}
|
||||
};
|
||||
|
||||
/// Used to make a "good" node for a highway graph with OSRM for low zooms.
|
||||
class Place
|
||||
{
|
||||
|
|
@ -272,7 +115,7 @@ private:
|
|||
double m_thresholdM;
|
||||
};
|
||||
|
||||
class MainFeaturesEmitter : public EmitterBase
|
||||
class MainFeaturesEmitter : public generator::EmitterBase
|
||||
{
|
||||
using TWorldGenerator = WorldMapGenerator<feature::FeaturesCollector>;
|
||||
using TCountriesGenerator = CountryMapGenerator<feature::Polygonizer<feature::FeaturesCollector>>;
|
||||
|
|
@ -579,8 +422,36 @@ private:
|
|||
}
|
||||
}
|
||||
};
|
||||
} // anonymous namespace
|
||||
} // namespace
|
||||
|
||||
namespace generator
|
||||
{
|
||||
// SourceReader ------------------------------------------------------------------------------------
|
||||
SourceReader::SourceReader() : m_file(unique_ptr<istream, Deleter>(&cin, Deleter(false)))
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from stdin"));
|
||||
}
|
||||
|
||||
SourceReader::SourceReader(string const & filename)
|
||||
: m_file(unique_ptr<istream, Deleter>(new ifstream(filename), Deleter()))
|
||||
{
|
||||
CHECK(static_cast<ifstream *>(m_file.get())->is_open(), ("Can't open file:", filename));
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from", filename));
|
||||
}
|
||||
|
||||
SourceReader::SourceReader(istringstream & stream)
|
||||
: m_file(unique_ptr<istream, Deleter>(&stream, Deleter(false)))
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Reading OSM data from memory"));
|
||||
}
|
||||
|
||||
uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
m_file->read(buffer, bufferSize);
|
||||
return m_file->gcount();
|
||||
}
|
||||
|
||||
// Functions ---------------------------------------------------------------------------------------
|
||||
unique_ptr<EmitterBase> MakeMainFeatureEmitter(feature::GenerateInfo const & info)
|
||||
{
|
||||
LOG(LINFO, ("Processing booking data from", info.m_bookingDatafileName, "done."));
|
||||
|
|
@ -669,21 +540,20 @@ void BuildIntermediateDataFromO5M(SourceReader & stream, TCache & cache, TownsDu
|
|||
|
||||
void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement *)> processor)
|
||||
{
|
||||
using TType = osm::O5MSource::EntityType;
|
||||
using Type = osm::O5MSource::EntityType;
|
||||
|
||||
osm::O5MSource dataset([&stream](uint8_t * buffer, size_t size)
|
||||
{
|
||||
return stream.Read(reinterpret_cast<char *>(buffer), size);
|
||||
});
|
||||
|
||||
auto translate = [](TType t) -> OsmElement::EntityType
|
||||
{
|
||||
auto translate = [](Type t) -> OsmElement::EntityType {
|
||||
switch (t)
|
||||
{
|
||||
case TType::Node: return OsmElement::EntityType::Node;
|
||||
case TType::Way: return OsmElement::EntityType::Way;
|
||||
case TType::Relation: return OsmElement::EntityType::Relation;
|
||||
default: return OsmElement::EntityType::Unknown;
|
||||
case Type::Node: return OsmElement::EntityType::Node;
|
||||
case Type::Way: return OsmElement::EntityType::Way;
|
||||
case Type::Relation: return OsmElement::EntityType::Relation;
|
||||
default: return OsmElement::EntityType::Unknown;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -694,21 +564,21 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement *
|
|||
|
||||
switch (em.type)
|
||||
{
|
||||
case TType::Node:
|
||||
case Type::Node:
|
||||
{
|
||||
p.type = OsmElement::EntityType::Node;
|
||||
p.lat = em.lat;
|
||||
p.lon = em.lon;
|
||||
break;
|
||||
}
|
||||
case TType::Way:
|
||||
case Type::Way:
|
||||
{
|
||||
p.type = OsmElement::EntityType::Way;
|
||||
for (uint64_t nd : em.Nodes())
|
||||
p.AddNd(nd);
|
||||
break;
|
||||
}
|
||||
case TType::Relation:
|
||||
case Type::Relation:
|
||||
{
|
||||
p.type = OsmElement::EntityType::Relation;
|
||||
for (auto const & member : em.Members())
|
||||
|
|
@ -731,15 +601,15 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement *
|
|||
|
||||
using PreEmit = function<bool(OsmElement *)>;
|
||||
|
||||
template <class TNodesHolder>
|
||||
template <class NodesHolder>
|
||||
bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter,
|
||||
PreEmit const & preEmit)
|
||||
{
|
||||
try
|
||||
{
|
||||
TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
|
||||
NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
|
||||
|
||||
using TDataCache = IntermediateData<TNodesHolder, cache::EMode::Read>;
|
||||
using TDataCache = cache::IntermediateDataReader<NodesHolder>;
|
||||
TDataCache cache(nodes, info);
|
||||
cache.LoadIndex();
|
||||
|
||||
|
|
@ -784,14 +654,13 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter,
|
|||
return true;
|
||||
}
|
||||
|
||||
template <class TNodesHolder>
|
||||
template <class NodesHolder>
|
||||
bool GenerateIntermediateDataImpl(feature::GenerateInfo & info)
|
||||
{
|
||||
try
|
||||
{
|
||||
TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
|
||||
using TDataCache = IntermediateData<TNodesHolder, cache::EMode::Write>;
|
||||
TDataCache cache(nodes, info);
|
||||
NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
|
||||
cache::IntermediateDataWriter<NodesHolder> cache(nodes, info);
|
||||
TownsDumper towns;
|
||||
|
||||
SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName);
|
||||
|
|
@ -810,11 +679,11 @@ bool GenerateIntermediateDataImpl(feature::GenerateInfo & info)
|
|||
|
||||
cache.SaveIndex();
|
||||
towns.Dump(info.GetIntermediateFileName(TOWNS_FILE, ""));
|
||||
LOG(LINFO, ("Added points count = ", nodes.GetProcessedPoint()));
|
||||
LOG(LINFO, ("Added points count =", nodes.GetNumProcessedPoints()));
|
||||
}
|
||||
catch (Writer::Exception const & e)
|
||||
{
|
||||
LOG(LCRITICAL, ("Error with file ", e.what()));
|
||||
LOG(LCRITICAL, ("Error with file:", e.what()));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -824,11 +693,11 @@ bool GenerateRaw(feature::GenerateInfo & info, std::unique_ptr<EmitterBase> emit
|
|||
switch (info.m_nodeStorageType)
|
||||
{
|
||||
case feature::GenerateInfo::NodeStorageType::File:
|
||||
return GenerateFeaturesImpl<cache::RawFilePointStorage<cache::EMode::Read>>(info, *emitter, preEmit);
|
||||
return GenerateFeaturesImpl<cache::RawFilePointStorageMmapReader>(info, *emitter, preEmit);
|
||||
case feature::GenerateInfo::NodeStorageType::Index:
|
||||
return GenerateFeaturesImpl<cache::MapFilePointStorage<cache::EMode::Read>>(info, *emitter, preEmit);
|
||||
return GenerateFeaturesImpl<cache::MapFilePointStorageReader>(info, *emitter, preEmit);
|
||||
case feature::GenerateInfo::NodeStorageType::Memory:
|
||||
return GenerateFeaturesImpl<cache::RawMemPointStorage<cache::EMode::Read>>(info, *emitter, preEmit);
|
||||
return GenerateFeaturesImpl<cache::RawMemPointStorageReader>(info, *emitter, preEmit);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -904,11 +773,12 @@ bool GenerateIntermediateData(feature::GenerateInfo & info)
|
|||
switch (info.m_nodeStorageType)
|
||||
{
|
||||
case feature::GenerateInfo::NodeStorageType::File:
|
||||
return GenerateIntermediateDataImpl<cache::RawFilePointStorage<cache::EMode::Write>>(info);
|
||||
return GenerateIntermediateDataImpl<cache::RawFilePointStorageWriter>(info);
|
||||
case feature::GenerateInfo::NodeStorageType::Index:
|
||||
return GenerateIntermediateDataImpl<cache::MapFilePointStorage<cache::EMode::Write>>(info);
|
||||
return GenerateIntermediateDataImpl<cache::MapFilePointStorageWriter>(info);
|
||||
case feature::GenerateInfo::NodeStorageType::Memory:
|
||||
return GenerateIntermediateDataImpl<cache::RawMemPointStorage<cache::EMode::Write>>(info);
|
||||
return GenerateIntermediateDataImpl<cache::RawMemPointStorageWriter>(info);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@
|
|||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class FeatureBuilder1;
|
||||
class FeatureParams;
|
||||
|
||||
namespace generator
|
||||
{
|
||||
class SourceReader
|
||||
{
|
||||
struct Deleter
|
||||
|
|
@ -33,10 +38,7 @@ public:
|
|||
uint64_t Read(char * buffer, uint64_t bufferSize);
|
||||
};
|
||||
|
||||
class FeatureBuilder1;
|
||||
class FeatureParams;
|
||||
|
||||
// Emitter is used in OsmElemen to FeatureBuilder translation process.
|
||||
// Emitter is used in OsmElement to FeatureBuilder translation process.
|
||||
class EmitterBase
|
||||
{
|
||||
public:
|
||||
|
|
@ -67,3 +69,4 @@ bool GenerateIntermediateData(feature::GenerateInfo & info);
|
|||
|
||||
void ProcessOsmElementsFromO5M(SourceReader & stream, std::function<void(OsmElement *)> processor);
|
||||
void ProcessOsmElementsFromXML(SourceReader & stream, std::function<void(OsmElement *)> processor);
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ DEFINE_string(out, "", "Output file path");
|
|||
|
||||
namespace
|
||||
{
|
||||
class Emitter : public EmitterBase
|
||||
class Emitter : public generator::EmitterBase
|
||||
{
|
||||
public:
|
||||
Emitter(std::vector<FeatureBuilder1> & features)
|
||||
|
|
@ -138,11 +138,10 @@ int main(int argc, char * argv[])
|
|||
classificator::Load();
|
||||
|
||||
auto info = GetGenerateInfo();
|
||||
GenerateIntermediateData(info);
|
||||
generator::GenerateIntermediateData(info);
|
||||
|
||||
std::vector<FeatureBuilder1> features;
|
||||
GenerateFeatures(info, [&features](feature::GenerateInfo const & /* info */)
|
||||
{
|
||||
generator::GenerateFeatures(info, [&features](feature::GenerateInfo const & /* info */) {
|
||||
return my::make_unique<Emitter>(features);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
34F5588C1DBF4C9600A4FC11 /* sponsored_scoring.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 34F558831DBF4C9600A4FC11 /* sponsored_scoring.cpp */; };
|
||||
34F5588D1DBF4C9600A4FC11 /* sponsored_scoring.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 34F558841DBF4C9600A4FC11 /* sponsored_scoring.hpp */; };
|
||||
34F5588F1DBF4C9F00A4FC11 /* restaurants_info.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 34F5588E1DBF4C9F00A4FC11 /* restaurants_info.cpp */; };
|
||||
3933ED0E20D7FD4E000B1526 /* intermediate_data.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */; };
|
||||
39B2B9781FB468CC00AB85A1 /* cities_boundaries_builder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 39B2B9761FB468CC00AB85A1 /* cities_boundaries_builder.cpp */; };
|
||||
39B2B9791FB468CC00AB85A1 /* cities_boundaries_builder.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 39B2B9771FB468CC00AB85A1 /* cities_boundaries_builder.hpp */; };
|
||||
3D51BC481D5E50F700F1FA8D /* centers_table_builder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3D51BC461D5E50F700F1FA8D /* centers_table_builder.cpp */; };
|
||||
|
|
@ -133,6 +134,7 @@
|
|||
34F558831DBF4C9600A4FC11 /* sponsored_scoring.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = sponsored_scoring.cpp; sourceTree = "<group>"; };
|
||||
34F558841DBF4C9600A4FC11 /* sponsored_scoring.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sponsored_scoring.hpp; sourceTree = "<group>"; };
|
||||
34F5588E1DBF4C9F00A4FC11 /* restaurants_info.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = restaurants_info.cpp; path = restaurants_info/restaurants_info.cpp; sourceTree = "<group>"; };
|
||||
3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = intermediate_data.cpp; sourceTree = "<group>"; };
|
||||
39B2B9761FB468CC00AB85A1 /* cities_boundaries_builder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = cities_boundaries_builder.cpp; sourceTree = "<group>"; };
|
||||
39B2B9771FB468CC00AB85A1 /* cities_boundaries_builder.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = cities_boundaries_builder.hpp; sourceTree = "<group>"; };
|
||||
3D51BC461D5E50F700F1FA8D /* centers_table_builder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = centers_table_builder.cpp; sourceTree = "<group>"; };
|
||||
|
|
@ -279,6 +281,7 @@
|
|||
6753401D1A3F2A1B00A0A8C3 /* generator */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */,
|
||||
9D1B3F1A2034624900278AC8 /* node_mixer.cpp */,
|
||||
9D1B3F192034624900278AC8 /* node_mixer.hpp */,
|
||||
40492BC42021DC53008E093A /* feature_helpers.cpp */,
|
||||
|
|
@ -566,6 +569,7 @@
|
|||
3D74EF071F86841C0081202C /* ugc_section_builder.cpp in Sources */,
|
||||
67C79BB11E2CEEAB00C40034 /* restriction_generator.cpp in Sources */,
|
||||
3D51BC521D5E512500F1FA8D /* altitude_generator.cpp in Sources */,
|
||||
3933ED0E20D7FD4E000B1526 /* intermediate_data.cpp in Sources */,
|
||||
670E7BB31EF9812B00A8E9ED /* metalines_builder.cpp in Sources */,
|
||||
3D51BC581D5E512500F1FA8D /* srtm_parser.cpp in Sources */,
|
||||
675340871A3F2A7400A0A8C3 /* unpack_mwm.cpp in Sources */,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue