From 053bbb0a5948be4f3ccebacbaf4689d1fd35fdae Mon Sep 17 00:00:00 2001 From: Maxim Pimenov Date: Mon, 18 Jun 2018 14:14:26 +0300 Subject: [PATCH] [generator] Separated readers and writers in IntermediateData. --- generator/CMakeLists.txt | 1 + .../source_to_element_test.cpp | 13 +- generator/intermediate_data.cpp | 250 +++++++ generator/intermediate_data.hpp | 657 +++++++++--------- generator/osm_source.cpp | 238 ++----- generator/osm_source.hpp | 11 +- .../restaurants_info/restaurants_info.cpp | 7 +- .../generator.xcodeproj/project.pbxproj | 4 + 8 files changed, 644 insertions(+), 537 deletions(-) create mode 100644 generator/intermediate_data.cpp diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index df1830cca2..e5134e7f57 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -39,6 +39,7 @@ set(SRC gen_mwm_info.hpp generate_info.hpp geometry_holder.hpp + intermediate_data.cpp intermediate_data.hpp intermediate_elements.hpp locality_sorter.cpp diff --git a/generator/generator_tests/source_to_element_test.cpp b/generator/generator_tests/source_to_element_test.cpp index eac95469b6..85f93031e5 100644 --- a/generator/generator_tests/source_to_element_test.cpp +++ b/generator/generator_tests/source_to_element_test.cpp @@ -1,12 +1,19 @@ #include "testing/testing.hpp" -#include "coding/parse_xml.hpp" -#include "generator/osm_source.hpp" +#include "generator/generator_tests/source_data.hpp" #include "generator/osm_element.hpp" +#include "generator/osm_source.hpp" +#include "coding/parse_xml.hpp" + +#include +#include #include +#include +#include +#include -#include "source_data.hpp" +using namespace generator; UNIT_TEST(Source_To_Element_create_from_xml_test) { diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp new file mode 100644 index 0000000000..90bd1cf18c --- /dev/null +++ b/generator/intermediate_data.cpp @@ -0,0 +1,250 @@ +#include "generator/intermediate_data.hpp" + +#include +#include + +#include "base/logging.hpp" + +#include "defines.hpp" + +using namespace std; + +namespace +{ +size_t const kFlushCount = 1024; +double const kValueOrder = 1e7; + +void ToLatLon(double lat, double lon, generator::cache::LatLon & ll) +{ + int64_t const lat64 = lat * kValueOrder; + int64_t const lon64 = lon * kValueOrder; + + ll.lat = static_cast(lat64); + ll.lon = static_cast(lon64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lon64, ("Longtitude is out of 32bit boundary!")); +} + +bool FromLatLon(generator::cache::LatLon const & ll, double & lat, double & lon) +{ + // Assume that a valid coordinate is not (0, 0). + if (ll.lat != 0.0 || ll.lon != 0.0) + { + lat = static_cast(ll.lat) / kValueOrder; + lon = static_cast(ll.lon) / kValueOrder; + return true; + } + lat = 0.0; + lon = 0.0; + return false; +} +} // namespace + +namespace generator +{ +namespace cache +{ +// IndexFileReader --------------------------------------------------------------------------------- +IndexFileReader::IndexFileReader(string const & name) : m_fileReader(name.c_str()) {} + +void IndexFileReader::ReadAll() +{ + m_elements.clear(); + size_t fileSize = m_fileReader.Size(); + if (fileSize == 0) + return; + + LOG_SHORT(LINFO, ("Offsets reading is started for file", m_fileReader.GetName())); + CHECK_EQUAL(0, fileSize % sizeof(Element), ("Damaged file.")); + + try + { + m_elements.resize(base::checked_cast(fileSize / sizeof(Element))); + } + catch (exception const &) // bad_alloc + { + LOG(LCRITICAL, ("Insufficient memory for required offset map")); + } + + m_fileReader.Read(0, &m_elements[0], base::checked_cast(fileSize)); + + sort(m_elements.begin(), m_elements.end(), ElementComparator()); + + LOG_SHORT(LINFO, ("Offsets reading is finished")); +} + +bool IndexFileReader::GetValueByKey(Key key, Value & value) const +{ + auto it = lower_bound(m_elements.begin(), m_elements.end(), key, ElementComparator()); + if (it != m_elements.end() && it->first == key) + { + value = it->second; + return true; + } + return false; +} + +// IndexFileWriter --------------------------------------------------------------------------------- +IndexFileWriter::IndexFileWriter(string const & name) : m_fileWriter(name.c_str()) {} + +void IndexFileWriter::WriteAll() +{ + if (m_elements.empty()) + return; + + m_fileWriter.Write(&m_elements[0], m_elements.size() * sizeof(Element)); + m_elements.clear(); +} + +void IndexFileWriter::Add(Key k, Value const & v) +{ + if (m_elements.size() > kFlushCount) + WriteAll(); + + m_elements.emplace_back(k, v); +} + +// OSMElementCacheReader --------------------------------------------------------------------------- +OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload) + : m_fileReader(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload) +{ + if (!m_preload) + return; + size_t sz = m_fileReader.Size(); + m_data.resize(sz); + m_fileReader.Read(0, m_data.data(), sz); +} + +void OSMElementCacheReader::LoadOffsets() { m_offsets.ReadAll(); } + +// OSMElementCacheWriter --------------------------------------------------------------------------- +OSMElementCacheWriter::OSMElementCacheWriter(string const & name, bool preload) + : m_fileWriter(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload) +{ +} + +void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); } + +// RawFilePointStorageMmapReader ------------------------------------------------------------------- +RawFilePointStorageMmapReader::RawFilePointStorageMmapReader(string const & name) + : m_mmapReader(name) +{ +} + +bool RawFilePointStorageMmapReader::GetPoint(uint64_t id, double & lat, double & lon) const +{ + LatLon ll; + m_mmapReader.Read(id * sizeof(ll), &ll, sizeof(ll)); + + bool ret = FromLatLon(ll, lat, lon); + if (!ret) + LOG(LERROR, ("Node with id =", id, " not found!")); + return ret; +} + +// RawFilePointStorageWriter ----------------------------------------------------------------------- +RawFilePointStorageWriter::RawFilePointStorageWriter(string const & name) : m_fileWriter(name) {} + +void RawFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon) +{ + LatLon ll; + ToLatLon(lat, lon, ll); + + m_fileWriter.Seek(id * sizeof(ll)); + m_fileWriter.Write(&ll, sizeof(ll)); + + ++m_numProcessedPoints; +} + +// RawMemPointStorageReader ------------------------------------------------------------------------ +RawMemPointStorageReader::RawMemPointStorageReader(string const & name) + : m_fileReader(name), m_data(static_cast(1) << 33) +{ + m_fileReader.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); +} + +bool RawMemPointStorageReader::GetPoint(uint64_t id, double & lat, double & lon) const +{ + LatLon const & ll = m_data[id]; + bool ret = FromLatLon(ll, lat, lon); + if (!ret) + LOG(LERROR, ("Node with id =", id, " not found!")); + return ret; +} + +// RawMemPointStorageWriter ------------------------------------------------------------------------ +RawMemPointStorageWriter::RawMemPointStorageWriter(string const & name) + : m_fileWriter(name), m_data(static_cast(1) << 33) +{ +} + +RawMemPointStorageWriter::~RawMemPointStorageWriter() +{ + m_fileWriter.Write(m_data.data(), m_data.size() * sizeof(LatLon)); +} + +void RawMemPointStorageWriter::AddPoint(uint64_t id, double lat, double lon) +{ + CHECK_LESS(id, m_data.size(), + ("Found node with id", id, "which is bigger than the allocated cache size")); + + LatLon & ll = m_data[id]; + ToLatLon(lat, lon, ll); + + ++m_numProcessedPoints; +} + +// MapFilePointStorageReader ----------------------------------------------------------------------- +MapFilePointStorageReader::MapFilePointStorageReader(string const & name) + : m_fileReader(name + ".short") +{ + LOG(LINFO, ("Nodes reading is started")); + + uint64_t const count = m_fileReader.Size(); + + uint64_t pos = 0; + while (pos < count) + { + LatLonPos ll; + m_fileReader.Read(pos, &ll, sizeof(ll)); + + m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); + + pos += sizeof(ll); + } + + LOG(LINFO, ("Nodes reading is finished")); +} + +bool MapFilePointStorageReader::GetPoint(uint64_t id, double & lat, double & lon) const +{ + auto i = m_map.find(id); + if (i == m_map.end()) + return false; + lat = static_cast(i->second.first) / kValueOrder; + lon = static_cast(i->second.second) / kValueOrder; + return true; +} + +// MapFilePointStorageWriter ----------------------------------------------------------------------- +MapFilePointStorageWriter::MapFilePointStorageWriter(string const & name) + : m_fileWriter(name + ".short") +{ +} + +void MapFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon) +{ + LatLon ll; + ToLatLon(lat, lon, ll); + + LatLonPos llp; + llp.pos = id; + llp.lat = ll.lat; + llp.lon = ll.lon; + + m_fileWriter.Write(&llp, sizeof(llp)); + + ++m_numProcessedPoints; +} +} // namespace cache +} // namespace generator diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index 7be66edf78..a5c899c4bf 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -1,5 +1,6 @@ #pragma once +#include "generator/generate_info.hpp" #include "generator/intermediate_elements.hpp" #include "coding/file_name_utils.hpp" @@ -10,9 +11,8 @@ #include "base/logging.hpp" #include -#include -#include -#include +#include +#include #include #include #include @@ -21,105 +21,45 @@ #include "defines.hpp" -/// Classes for reading and writing any data in file with map of offsets for -/// fast searching in memory by some key. +// Classes for reading and writing any data in file with map of offsets for +// fast searching in memory by some key. +namespace generator +{ namespace cache { +using Key = uint64_t; -enum class EMode { Write = true, Read = false }; - -namespace detail +// Used to store all world nodes inside temporary index file. +// To find node by id, just calculate offset inside index file: +// offset_in_file = sizeof(LatLon) * node_ID +struct LatLon { -template -class IndexFile + int32_t lat; + int32_t lon; +}; +static_assert(sizeof(LatLon) == 8, "Invalid structure size"); + +struct LatLonPos { - using TKey = uint64_t; - static_assert(is_integral::value, "TKey is not integral type"); - using TElement = std::pair; - using TContainer = std::vector; - - TContainer m_elements; - TFile m_file; - - static size_t constexpr kFlushCount = 1024; - - struct ElementComparator - { - bool operator()(TElement const & r1, TElement const & r2) const - { - return ((r1.first == r2.first) ? r1.second < r2.second : r1.first < r2.first); - } - bool operator()(TElement const & r1, TKey r2) const { return (r1.first < r2); } - bool operator()(TKey r1, TElement const & r2) const { return (r1 < r2.first); } - }; - - static size_t CheckedCast(uint64_t v) - { - ASSERT_LESS(v, std::numeric_limits::max(), ("Value too long for memory address : ", v)); - return static_cast(v); - } + uint64_t pos; + int32_t lat; + int32_t lon; +}; +static_assert(sizeof(LatLonPos) == 16, "Invalid structure size"); +class IndexFileReader +{ public: - explicit IndexFile(std::string const & name) : m_file(name.c_str()) {} + using Value = uint64_t; - std::string GetFileName() const { return m_file.GetName(); } + explicit IndexFileReader(std::string const & name); - void WriteAll() - { - if (m_elements.empty()) - return; + void ReadAll(); - m_file.Write(&m_elements[0], m_elements.size() * sizeof(TElement)); - m_elements.clear(); - } + bool GetValueByKey(Key key, Value & value) const; - void ReadAll() - { - m_elements.clear(); - size_t fileSize = m_file.Size(); - if (fileSize == 0) - return; - - LOG_SHORT(LINFO, ("Offsets reading is started for file ", GetFileName())); - CHECK_EQUAL(0, fileSize % sizeof(TElement), ("Damaged file.")); - - try - { - m_elements.resize(CheckedCast(fileSize / sizeof(TElement))); - } - catch (std::exception const &) // bad_alloc - { - LOG(LCRITICAL, ("Insufficient memory for required offset map")); - } - - m_file.Read(0, &m_elements[0], CheckedCast(fileSize)); - - std::sort(m_elements.begin(), m_elements.end(), ElementComparator()); - - LOG_SHORT(LINFO, ("Offsets reading is finished")); - } - - void Add(TKey k, TValue const & v) - { - if (m_elements.size() > kFlushCount) - WriteAll(); - - m_elements.push_back(std::make_pair(k, v)); - } - - bool GetValueByKey(TKey key, TValue & value) const - { - auto it = std::lower_bound(m_elements.begin(), m_elements.end(), key, ElementComparator()); - if ((it != m_elements.end()) && ((*it).first == key)) - { - value = (*it).second; - return true; - } - return false; - } - - template - void ForEachByKey(TKey k, ToDo && toDo) const + template + void ForEachByKey(Key k, ToDo && toDo) const { auto range = std::equal_range(m_elements.begin(), m_elements.end(), k, ElementComparator()); for (; range.first != range.second; ++range.first) @@ -128,73 +68,55 @@ public: return; } } -}; -} // namespace detail -template -class OSMElementCache +private: + static_assert(is_integral::value, "Key must be an integral type"); + using Element = std::pair; + + struct ElementComparator + { + bool operator()(Element const & r1, Element const & r2) const + { + return ((r1.first == r2.first) ? r1.second < r2.second : r1.first < r2.first); + } + bool operator()(Element const & r1, Key r2) const { return (r1.first < r2); } + bool operator()(Key r1, Element const & r2) const { return (r1 < r2.first); } + }; + + std::vector m_elements; + FileReader m_fileReader; +}; + +class IndexFileWriter { public: - using TKey = uint64_t; - using TStorage = std::conditional_t; - using TOffsetFile = std::conditional_t; + using Value = uint64_t; -protected: - using TBuffer = std::vector; - TStorage m_storage; - detail::IndexFile m_offsets; - std::string m_name; - TBuffer m_data; - bool m_preload = false; + explicit IndexFileWriter(std::string const & name); + void WriteAll(); + + void Add(Key k, Value const & v); + +private: + using Element = std::pair; + + std::vector m_elements; + FileWriter m_fileWriter; +}; + +class OSMElementCacheReader +{ public: - OSMElementCache(std::string const & name, bool preload = false) - : m_storage(name) - , m_offsets(name + OFFSET_EXT) - , m_name(name) - , m_preload(preload) - { - InitStorage(); - } + OSMElementCacheReader(std::string const & name, bool preload = false); - template - std::enable_if_t InitStorage() - { - } - - template - std::enable_if_t InitStorage() - { - if (!m_preload) - return; - size_t sz = m_storage.Size(); - m_data.resize(sz); - m_storage.Read(0, m_data.data(), sz); - } - - template - std::enable_if_t Write(TKey id, TValue const & value) - { - m_offsets.Add(id, m_storage.Pos()); - m_data.clear(); - MemWriter w(m_data); - - value.Write(w); - - // write buffer - ASSERT_LESS(m_data.size(), std::numeric_limits::max(), ()); - uint32_t sz = static_cast(m_data.size()); - m_storage.Write(&sz, sizeof(sz)); - m_storage.Write(m_data.data(), sz * sizeof(TBuffer::value_type)); - } - - template - std::enable_if_t Read(TKey id, TValue & value) + template + bool Read(Key id, Value & value) { uint64_t pos = 0; if (!m_offsets.GetValueByKey(id, pos)) { - LOG_SHORT(LWARNING, ("Can't find offset in file", m_offsets.GetFileName(), "by id", id)); + LOG_SHORT(LWARNING, ("Can't find offset in file", m_name + OFFSET_EXT, "by id", id)); return false; } @@ -204,9 +126,9 @@ public: if (!m_preload) { // in case not-in-memory work we read buffer - m_storage.Read(pos, &valueSize, sizeof(valueSize)); + m_fileReader.Read(pos, &valueSize, sizeof(valueSize)); m_data.resize(valueSize); - m_storage.Read(pos + sizeof(valueSize), m_data.data(), valueSize); + m_fileReader.Read(pos + sizeof(valueSize), m_data.data(), valueSize); offset = 0; } @@ -215,222 +137,273 @@ public: return true; } - inline void SaveOffsets() { m_offsets.WriteAll(); } - inline void LoadOffsets() { m_offsets.ReadAll(); } + void LoadOffsets(); + +protected: + FileReader m_fileReader; + IndexFileReader m_offsets; + std::string m_name; + std::vector m_data; + bool m_preload = false; }; -/// Used to store all world nodes inside temporary index file. -/// To find node by id, just calculate offset inside index file: -/// offset_in_file = sizeof(LatLon) * node_ID -class PointStorage +class OSMElementCacheWriter { - size_t m_processedPoint = 0; - public: - struct LatLon + OSMElementCacheWriter(std::string const & name, bool preload = false); + + template + void Write(Key id, Value const & value) { - int32_t lat; - int32_t lon; - }; - static_assert(sizeof(LatLon) == 8, "Invalid structure size"); + m_offsets.Add(id, m_fileWriter.Pos()); + m_data.clear(); + MemWriter w(m_data); - struct LatLonPos - { - uint64_t pos; - int32_t lat; - int32_t lon; - }; - static_assert(sizeof(LatLonPos) == 16, "Invalid structure size"); + value.Write(w); - inline size_t GetProcessedPoint() const { return m_processedPoint; } - inline void IncProcessedPoint() { ++m_processedPoint; } -}; - -template -class RawFilePointStorage : public PointStorage -{ -#ifdef OMIM_OS_WINDOWS - using TFileReader = FileReader; -#else - using TFileReader = MmapReader; -#endif - - std::conditional_t m_file; - - constexpr static double const kValueOrder = 1E+7; - -public: - explicit RawFilePointStorage(std::string const & name) : m_file(name) {} - - template - std::enable_if_t AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - LatLon ll; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - - m_file.Seek(id * sizeof(ll)); - m_file.Write(&ll, sizeof(ll)); - - IncProcessedPoint(); + ASSERT_LESS(m_data.size(), std::numeric_limits::max(), ()); + uint32_t sz = static_cast(m_data.size()); + // ?! + m_fileWriter.Write(&sz, sizeof(sz)); + m_fileWriter.Write(m_data.data(), sz); } - template - std::enable_if_t GetPoint(uint64_t id, double & lat, double & lng) const - { - LatLon ll; - m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); + void SaveOffsets(); - // assume that valid coordinate is not (0, 0) - if (ll.lat != 0.0 || ll.lon != 0.0) - { - lat = static_cast(ll.lat) / kValueOrder; - lng = static_cast(ll.lon) / kValueOrder; - return true; - } - LOG(LERROR, ("Node with id = ", id, " not found!")); - return false; - } +protected: + FileWriter m_fileWriter; + IndexFileWriter m_offsets; + std::string m_name; + std::vector m_data; + bool m_preload = false; }; -template -class RawMemPointStorage : public PointStorage +class RawFilePointStorageMmapReader { - std::conditional_t m_file; +public: + explicit RawFilePointStorageMmapReader(std::string const & name); - constexpr static double const kValueOrder = 1E+7; + bool GetPoint(uint64_t id, double & lat, double & lon) const; +private: + MmapReader m_mmapReader; +}; + +class RawFilePointStorageWriter +{ +public: + explicit RawFilePointStorageWriter(std::string const & name); + + void AddPoint(uint64_t id, double lat, double lon); + uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; } + +private: + FileWriter m_fileWriter; + uint64_t m_numProcessedPoints = 0; +}; + +class RawMemPointStorageReader +{ +public: + explicit RawMemPointStorageReader(std::string const & name); + + bool GetPoint(uint64_t id, double & lat, double & lon) const; + +private: + FileReader m_fileReader; std::vector m_data; - -public: - explicit RawMemPointStorage(std::string const & name) : m_file(name), m_data(static_cast(1) << 33) - { - InitStorage(); - } - - ~RawMemPointStorage() { DoneStorage(); } - - template - std::enable_if_t InitStorage() - { - } - - template - std::enable_if_t InitStorage() - { - m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); - } - - template - std::enable_if_t DoneStorage() - { - m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon)); - } - - template - std::enable_if_t DoneStorage() - { - } - - template - std::enable_if_t AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - CHECK_LESS(id, m_data.size(), ("Found node with id", id, "which is bigger than the allocated cache size")); - LatLon & ll = m_data[id]; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - - IncProcessedPoint(); - } - - template - std::enable_if_t GetPoint(uint64_t id, double & lat, double & lng) const - { - LatLon const & ll = m_data[id]; - // assume that valid coordinate is not (0, 0) - if (ll.lat != 0.0 || ll.lon != 0.0) - { - lat = static_cast(ll.lat) / kValueOrder; - lng = static_cast(ll.lon) / kValueOrder; - return true; - } - LOG(LERROR, ("Node with id = ", id, " not found!")); - return false; - } }; -template -class MapFilePointStorage : public PointStorage +class RawMemPointStorageWriter { - std::conditional_t m_file; - std::unordered_map> m_map; - - constexpr static double const kValueOrder = 1E+7; - public: - explicit MapFilePointStorage(std::string const & name) : m_file(name + ".short") { InitStorage(); } + explicit RawMemPointStorageWriter(std::string const & name); - template - std::enable_if_t InitStorage() - { - } + ~RawMemPointStorageWriter(); - template - std::enable_if_t InitStorage() - { - LOG(LINFO, ("Nodes reading is started")); + void AddPoint(uint64_t id, double lat, double lon); + uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; } - uint64_t const count = m_file.Size(); - - uint64_t pos = 0; - while (pos < count) - { - LatLonPos ll; - m_file.Read(pos, &ll, sizeof(ll)); - - m_map.emplace(std::make_pair(ll.pos, std::make_pair(ll.lat, ll.lon))); - - pos += sizeof(ll); - } - - LOG(LINFO, ("Nodes reading is finished")); - } - - void AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - LatLonPos ll; - ll.pos = id; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - m_file.Write(&ll, sizeof(ll)); - - IncProcessedPoint(); - } - - bool GetPoint(uint64_t id, double & lat, double & lng) const - { - auto i = m_map.find(id); - if (i == m_map.end()) - return false; - lat = static_cast(i->second.first) / kValueOrder; - lng = static_cast(i->second.second) / kValueOrder; - return true; - } +private: + FileWriter m_fileWriter; + std::vector m_data; + uint64_t m_numProcessedPoints = 0; }; +class MapFilePointStorageReader +{ +public: + explicit MapFilePointStorageReader(std::string const & name); + + bool GetPoint(uint64_t id, double & lat, double & lon) const; + +private: + FileReader m_fileReader; + std::unordered_map> m_map; +}; + +class MapFilePointStorageWriter +{ +public: + explicit MapFilePointStorageWriter(std::string const & name); + + void AddPoint(uint64_t id, double lat, double lon); + uint64_t GetNumProcessedPoints() { return m_numProcessedPoints; } + +private: + FileWriter m_fileWriter; + std::unordered_map> m_map; + uint64_t m_numProcessedPoints = 0; +}; + +template +class IntermediateDataReader +{ +public: + IntermediateDataReader(NodesHolder & nodes, feature::GenerateInfo & info) + : m_nodes(nodes) + , m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache) + , m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache) + , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) + , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) + { + } + + bool GetNode(Key id, double & lat, double & lon) { return m_nodes.GetPoint(id, lat, lon); } + bool GetWay(Key id, WayElement & e) { return m_ways.Read(id, e); } + + template + void ForEachRelationByWay(Key id, ToDo && toDo) + { + RelationProcessor processor(m_relations, toDo); + m_wayToRelations.ForEachByKey(id, processor); + } + + template + void ForEachRelationByWayCached(Key id, ToDo && toDo) + { + CachedRelationProcessor processor(m_relations, toDo); + m_wayToRelations.ForEachByKey(id, processor); + } + + template + void ForEachRelationByNodeCached(Key id, ToDo && toDo) + { + CachedRelationProcessor processor(m_relations, toDo); + m_nodeToRelations.ForEachByKey(id, processor); + } + + void LoadIndex() + { + m_ways.LoadOffsets(); + m_relations.LoadOffsets(); + + m_nodeToRelations.ReadAll(); + m_wayToRelations.ReadAll(); + } + +private: + using CacheReader = cache::OSMElementCacheReader; + + template + struct ElementProcessorBase + { + protected: + CacheReader m_reader; + ToDo & m_toDo; + + public: + ElementProcessorBase(CacheReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} + + bool operator()(uint64_t id) + { + Element e; + return m_reader.Read(id, e) ? m_toDo(id, e) : false; + } + }; + + template + struct RelationProcessor : public ElementProcessorBase + { + using Base = ElementProcessorBase; + + RelationProcessor(CacheReader & reader, ToDo & toDo) : Base(reader, toDo) {} + }; + + template + struct CachedRelationProcessor : public RelationProcessor + { + using Base = RelationProcessor; + + CachedRelationProcessor(CacheReader & reader, ToDo & toDo) : Base(reader, toDo) {} + bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } + }; + + NodesHolder & m_nodes; + + cache::OSMElementCacheReader m_ways; + cache::OSMElementCacheReader m_relations; + + cache::IndexFileReader m_nodeToRelations; + cache::IndexFileReader m_wayToRelations; +}; + +template +class IntermediateDataWriter +{ +public: + IntermediateDataWriter(NodesHolder & nodes, feature::GenerateInfo & info) + : m_nodes(nodes) + , m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache) + , m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache) + , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) + , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) + { + } + + void AddNode(Key id, double lat, double lon) { m_nodes.AddPoint(id, lat, lon); } + + void AddWay(Key id, WayElement const & e) { m_ways.Write(id, e); } + + void AddRelation(Key id, RelationElement const & e) + { + string const & relationType = e.GetType(); + if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" || + relationType == "associatedStreet" || relationType == "building" || + relationType == "restriction")) + { + return; + } + + m_relations.Write(id, e); + AddToIndex(m_nodeToRelations, id, e.nodes); + AddToIndex(m_wayToRelations, id, e.ways); + } + + void SaveIndex() + { + m_ways.SaveOffsets(); + m_relations.SaveOffsets(); + + m_nodeToRelations.WriteAll(); + m_wayToRelations.WriteAll(); + } + +private: + NodesHolder & m_nodes; + + cache::OSMElementCacheWriter m_ways; + cache::OSMElementCacheWriter m_relations; + + cache::IndexFileWriter m_nodeToRelations; + cache::IndexFileWriter m_wayToRelations; + + template + static void AddToIndex(Index & index, Key relationId, Container const & values) + { + for (auto const & v : values) + index.Add(v.first, relationId); + } +}; } // namespace cache +} // namespace generator diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 81dc3b2053..0604e68c8b 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -39,165 +39,8 @@ using namespace std; -SourceReader::SourceReader() -: m_file(unique_ptr(&cin, Deleter(false))) -{ - LOG_SHORT(LINFO, ("Reading OSM data from stdin")); -} - -SourceReader::SourceReader(string const & filename) -: m_file(unique_ptr(new ifstream(filename), Deleter())) -{ - CHECK(static_cast(m_file.get())->is_open() , ("Can't open file:", filename)); - LOG_SHORT(LINFO, ("Reading OSM data from", filename)); -} - -SourceReader::SourceReader(istringstream & stream) -: m_file(unique_ptr(&stream, Deleter(false))) -{ - LOG_SHORT(LINFO, ("Reading OSM data from memory")); -} - -uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize) -{ - m_file->read(buffer, bufferSize); - return m_file->gcount(); -} - namespace { -template -class IntermediateData -{ - using TReader = cache::OSMElementCache; - - using TFile = conditional_t; - - using TKey = uint64_t; - static_assert(is_integral::value, "TKey is not integral type"); - - using TIndex = cache::detail::IndexFile; - - TNodesHolder & m_nodes; - - TReader m_ways; - TReader m_relations; - - TIndex m_nodeToRelations; - TIndex m_wayToRelations; - - template - struct ElementProcessorBase - { - protected: - TReader & m_reader; - ToDo & m_toDo; - - public: - ElementProcessorBase(TReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} - - bool operator()(uint64_t id) - { - TElement e; - return m_reader.Read(id, e) ? m_toDo(id, e) : false; - } - }; - - template - struct RelationProcessor : public ElementProcessorBase - { - using TBase = ElementProcessorBase; - - RelationProcessor(TReader & reader, ToDo & toDo) : TBase(reader, toDo) {} - }; - - template - struct CachedRelationProcessor : public RelationProcessor - { - using TBase = RelationProcessor; - - CachedRelationProcessor(TReader & rels, ToDo & toDo) : TBase(rels, toDo) {} - bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } - }; - - template - static void AddToIndex(TIndex & index, TKey relationId, TContainer const & values) - { - for (auto const & v : values) - index.Add(v.first, relationId); - } - -public: - IntermediateData(TNodesHolder & nodes, feature::GenerateInfo & info) - : m_nodes(nodes) - , m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache) - , m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache) - , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) - , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE,ID2REL_EXT)) - { - } - - void AddNode(TKey id, double lat, double lng) { m_nodes.AddPoint(id, lat, lng); } - bool GetNode(TKey id, double & lat, double & lng) { return m_nodes.GetPoint(id, lat, lng); } - - void AddWay(TKey id, WayElement const & e) { m_ways.Write(id, e); } - bool GetWay(TKey id, WayElement & e) { return m_ways.Read(id, e); } - - void AddRelation(TKey id, RelationElement const & e) - { - string const & relationType = e.GetType(); - if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" || - relationType == "associatedStreet" || relationType == "building" || - relationType == "restriction")) - { - return; - } - - m_relations.Write(id, e); - AddToIndex(m_nodeToRelations, id, e.nodes); - AddToIndex(m_wayToRelations, id, e.ways); - } - - template - void ForEachRelationByWay(TKey id, ToDo && toDo) - { - RelationProcessor processor(m_relations, toDo); - m_wayToRelations.ForEachByKey(id, processor); - } - - template - void ForEachRelationByNodeCached(TKey id, ToDo && toDo) - { - CachedRelationProcessor processor(m_relations, toDo); - m_nodeToRelations.ForEachByKey(id, processor); - } - - template - void ForEachRelationByWayCached(TKey id, ToDo && toDo) - { - CachedRelationProcessor processor(m_relations, toDo); - m_wayToRelations.ForEachByKey(id, processor); - } - - void SaveIndex() - { - m_ways.SaveOffsets(); - m_relations.SaveOffsets(); - - m_nodeToRelations.WriteAll(); - m_wayToRelations.WriteAll(); - } - - void LoadIndex() - { - m_ways.LoadOffsets(); - m_relations.LoadOffsets(); - - m_nodeToRelations.ReadAll(); - m_wayToRelations.ReadAll(); - } -}; - /// Used to make a "good" node for a highway graph with OSRM for low zooms. class Place { @@ -272,7 +115,7 @@ private: double m_thresholdM; }; -class MainFeaturesEmitter : public EmitterBase +class MainFeaturesEmitter : public generator::EmitterBase { using TWorldGenerator = WorldMapGenerator; using TCountriesGenerator = CountryMapGenerator>; @@ -579,8 +422,36 @@ private: } } }; -} // anonymous namespace +} // namespace +namespace generator +{ +// SourceReader ------------------------------------------------------------------------------------ +SourceReader::SourceReader() : m_file(unique_ptr(&cin, Deleter(false))) +{ + LOG_SHORT(LINFO, ("Reading OSM data from stdin")); +} + +SourceReader::SourceReader(string const & filename) + : m_file(unique_ptr(new ifstream(filename), Deleter())) +{ + CHECK(static_cast(m_file.get())->is_open(), ("Can't open file:", filename)); + LOG_SHORT(LINFO, ("Reading OSM data from", filename)); +} + +SourceReader::SourceReader(istringstream & stream) + : m_file(unique_ptr(&stream, Deleter(false))) +{ + LOG_SHORT(LINFO, ("Reading OSM data from memory")); +} + +uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize) +{ + m_file->read(buffer, bufferSize); + return m_file->gcount(); +} + +// Functions --------------------------------------------------------------------------------------- unique_ptr MakeMainFeatureEmitter(feature::GenerateInfo const & info) { LOG(LINFO, ("Processing booking data from", info.m_bookingDatafileName, "done.")); @@ -669,21 +540,20 @@ void BuildIntermediateDataFromO5M(SourceReader & stream, TCache & cache, TownsDu void ProcessOsmElementsFromO5M(SourceReader & stream, function processor) { - using TType = osm::O5MSource::EntityType; + using Type = osm::O5MSource::EntityType; osm::O5MSource dataset([&stream](uint8_t * buffer, size_t size) { return stream.Read(reinterpret_cast(buffer), size); }); - auto translate = [](TType t) -> OsmElement::EntityType - { + auto translate = [](Type t) -> OsmElement::EntityType { switch (t) { - case TType::Node: return OsmElement::EntityType::Node; - case TType::Way: return OsmElement::EntityType::Way; - case TType::Relation: return OsmElement::EntityType::Relation; - default: return OsmElement::EntityType::Unknown; + case Type::Node: return OsmElement::EntityType::Node; + case Type::Way: return OsmElement::EntityType::Way; + case Type::Relation: return OsmElement::EntityType::Relation; + default: return OsmElement::EntityType::Unknown; } }; @@ -694,21 +564,21 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function; -template +template bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter, PreEmit const & preEmit) { try { - TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); + NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - using TDataCache = IntermediateData; + using TDataCache = cache::IntermediateDataReader; TDataCache cache(nodes, info); cache.LoadIndex(); @@ -784,14 +654,13 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter, return true; } -template +template bool GenerateIntermediateDataImpl(feature::GenerateInfo & info) { try { - TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - using TDataCache = IntermediateData; - TDataCache cache(nodes, info); + NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); + cache::IntermediateDataWriter cache(nodes, info); TownsDumper towns; SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); @@ -810,11 +679,11 @@ bool GenerateIntermediateDataImpl(feature::GenerateInfo & info) cache.SaveIndex(); towns.Dump(info.GetIntermediateFileName(TOWNS_FILE, "")); - LOG(LINFO, ("Added points count = ", nodes.GetProcessedPoint())); + LOG(LINFO, ("Added points count =", nodes.GetNumProcessedPoints())); } catch (Writer::Exception const & e) { - LOG(LCRITICAL, ("Error with file ", e.what())); + LOG(LCRITICAL, ("Error with file:", e.what())); } return true; } @@ -824,11 +693,11 @@ bool GenerateRaw(feature::GenerateInfo & info, std::unique_ptr emit switch (info.m_nodeStorageType) { case feature::GenerateInfo::NodeStorageType::File: - return GenerateFeaturesImpl>(info, *emitter, preEmit); + return GenerateFeaturesImpl(info, *emitter, preEmit); case feature::GenerateInfo::NodeStorageType::Index: - return GenerateFeaturesImpl>(info, *emitter, preEmit); + return GenerateFeaturesImpl(info, *emitter, preEmit); case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateFeaturesImpl>(info, *emitter, preEmit); + return GenerateFeaturesImpl(info, *emitter, preEmit); } return false; } @@ -904,11 +773,12 @@ bool GenerateIntermediateData(feature::GenerateInfo & info) switch (info.m_nodeStorageType) { case feature::GenerateInfo::NodeStorageType::File: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl(info); case feature::GenerateInfo::NodeStorageType::Index: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl(info); case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl(info); } return false; } +} // namespace generator diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp index e627ad581e..eb820cb1fd 100644 --- a/generator/osm_source.hpp +++ b/generator/osm_source.hpp @@ -10,6 +10,11 @@ #include #include +class FeatureBuilder1; +class FeatureParams; + +namespace generator +{ class SourceReader { struct Deleter @@ -33,10 +38,7 @@ public: uint64_t Read(char * buffer, uint64_t bufferSize); }; -class FeatureBuilder1; -class FeatureParams; - -// Emitter is used in OsmElemen to FeatureBuilder translation process. +// Emitter is used in OsmElement to FeatureBuilder translation process. class EmitterBase { public: @@ -67,3 +69,4 @@ bool GenerateIntermediateData(feature::GenerateInfo & info); void ProcessOsmElementsFromO5M(SourceReader & stream, std::function processor); void ProcessOsmElementsFromXML(SourceReader & stream, std::function processor); +} // namespace generator diff --git a/generator/restaurants_info/restaurants_info.cpp b/generator/restaurants_info/restaurants_info.cpp index a67d0c3cb5..e57271bf8b 100644 --- a/generator/restaurants_info/restaurants_info.cpp +++ b/generator/restaurants_info/restaurants_info.cpp @@ -25,7 +25,7 @@ DEFINE_string(out, "", "Output file path"); namespace { -class Emitter : public EmitterBase +class Emitter : public generator::EmitterBase { public: Emitter(std::vector & features) @@ -138,11 +138,10 @@ int main(int argc, char * argv[]) classificator::Load(); auto info = GetGenerateInfo(); - GenerateIntermediateData(info); + generator::GenerateIntermediateData(info); std::vector features; - GenerateFeatures(info, [&features](feature::GenerateInfo const & /* info */) - { + generator::GenerateFeatures(info, [&features](feature::GenerateInfo const & /* info */) { return my::make_unique(features); }); diff --git a/xcode/generator/generator.xcodeproj/project.pbxproj b/xcode/generator/generator.xcodeproj/project.pbxproj index a85bb9818a..4ffaaea2b5 100644 --- a/xcode/generator/generator.xcodeproj/project.pbxproj +++ b/xcode/generator/generator.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 34F5588C1DBF4C9600A4FC11 /* sponsored_scoring.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 34F558831DBF4C9600A4FC11 /* sponsored_scoring.cpp */; }; 34F5588D1DBF4C9600A4FC11 /* sponsored_scoring.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 34F558841DBF4C9600A4FC11 /* sponsored_scoring.hpp */; }; 34F5588F1DBF4C9F00A4FC11 /* restaurants_info.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 34F5588E1DBF4C9F00A4FC11 /* restaurants_info.cpp */; }; + 3933ED0E20D7FD4E000B1526 /* intermediate_data.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */; }; 39B2B9781FB468CC00AB85A1 /* cities_boundaries_builder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 39B2B9761FB468CC00AB85A1 /* cities_boundaries_builder.cpp */; }; 39B2B9791FB468CC00AB85A1 /* cities_boundaries_builder.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 39B2B9771FB468CC00AB85A1 /* cities_boundaries_builder.hpp */; }; 3D51BC481D5E50F700F1FA8D /* centers_table_builder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3D51BC461D5E50F700F1FA8D /* centers_table_builder.cpp */; }; @@ -133,6 +134,7 @@ 34F558831DBF4C9600A4FC11 /* sponsored_scoring.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = sponsored_scoring.cpp; sourceTree = ""; }; 34F558841DBF4C9600A4FC11 /* sponsored_scoring.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sponsored_scoring.hpp; sourceTree = ""; }; 34F5588E1DBF4C9F00A4FC11 /* restaurants_info.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = restaurants_info.cpp; path = restaurants_info/restaurants_info.cpp; sourceTree = ""; }; + 3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = intermediate_data.cpp; sourceTree = ""; }; 39B2B9761FB468CC00AB85A1 /* cities_boundaries_builder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = cities_boundaries_builder.cpp; sourceTree = ""; }; 39B2B9771FB468CC00AB85A1 /* cities_boundaries_builder.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = cities_boundaries_builder.hpp; sourceTree = ""; }; 3D51BC461D5E50F700F1FA8D /* centers_table_builder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = centers_table_builder.cpp; sourceTree = ""; }; @@ -279,6 +281,7 @@ 6753401D1A3F2A1B00A0A8C3 /* generator */ = { isa = PBXGroup; children = ( + 3933ED0D20D7FD4E000B1526 /* intermediate_data.cpp */, 9D1B3F1A2034624900278AC8 /* node_mixer.cpp */, 9D1B3F192034624900278AC8 /* node_mixer.hpp */, 40492BC42021DC53008E093A /* feature_helpers.cpp */, @@ -566,6 +569,7 @@ 3D74EF071F86841C0081202C /* ugc_section_builder.cpp in Sources */, 67C79BB11E2CEEAB00C40034 /* restriction_generator.cpp in Sources */, 3D51BC521D5E512500F1FA8D /* altitude_generator.cpp in Sources */, + 3933ED0E20D7FD4E000B1526 /* intermediate_data.cpp in Sources */, 670E7BB31EF9812B00A8E9ED /* metalines_builder.cpp in Sources */, 3D51BC581D5E512500F1FA8D /* srtm_parser.cpp in Sources */, 675340871A3F2A7400A0A8C3 /* unpack_mwm.cpp in Sources */,