From 067ad8ef36eb7fcba129c4312ba2d2da0cb51b1f Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Tue, 13 Aug 2019 18:11:50 +0300 Subject: [PATCH] [generator] Added cloneable cache. --- generator/intermediate_data.cpp | 134 +++++++++++++++++++++----------- generator/intermediate_data.hpp | 71 ++++++++++++----- 2 files changed, 142 insertions(+), 63 deletions(-) diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index b4dac2e01b..24af22cc6b 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -33,11 +33,11 @@ void ToLatLon(double lat, double lon, LatLon & ll) int64_t const lon64 = lon * kValueOrder; CHECK( - lat64 >= std::numeric_limits::min() && lat64 <= std::numeric_limits::max(), - ("Latitude is out of 32bit boundary:", lat64)); + lat64 >= numeric_limits::min() && lat64 <= numeric_limits::max(), + ("Latitude is out of 32bit boundary:", lat64)); CHECK( - lon64 >= std::numeric_limits::min() && lon64 <= std::numeric_limits::max(), - ("Longitude is out of 32bit boundary:", lon64)); + lon64 >= numeric_limits::min() && lon64 <= numeric_limits::max(), + ("Longitude is out of 32bit boundary:", lon64)); ll.m_lat = static_cast(lat64); ll.m_lon = static_cast(lon64); } @@ -261,16 +261,15 @@ private: } // namespace // IndexFileReader --------------------------------------------------------------------------------- -IndexFileReader::IndexFileReader(string const & name) : m_fileReader(name.c_str()) {} - -void IndexFileReader::ReadAll() +IndexFileReader::IndexFileReader(string const & name) { + FileReader fileReader(name.c_str()); m_elements.clear(); - size_t fileSize = m_fileReader.Size(); + size_t fileSize = fileReader.Size(); if (fileSize == 0) return; - LOG_SHORT(LINFO, ("Offsets reading is started for file", m_fileReader.GetName())); + LOG_SHORT(LINFO, ("Offsets reading is started for file", fileReader.GetName())); CHECK_EQUAL(0, fileSize % sizeof(Element), ("Damaged file.")); try @@ -282,7 +281,7 @@ void IndexFileReader::ReadAll() LOG(LCRITICAL, ("Insufficient memory for required offset map")); } - m_fileReader.Read(0, &m_elements[0], base::checked_cast(fileSize)); + fileReader.Read(0, &m_elements[0], base::checked_cast(fileSize)); sort(m_elements.begin(), m_elements.end(), ElementComparator()); @@ -324,8 +323,11 @@ void IndexFileWriter::Add(Key k, Value const & v) } // OSMElementCacheReader --------------------------------------------------------------------------- -OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload) - : m_fileReader(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload) +OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload, bool forceReload) + : m_fileReader(name) + , m_offsets(IndexReader::GetOrCreate(name + OFFSET_EXT, forceReload)) + , m_name(name) + , m_preload(preload) { if (!m_preload) return; @@ -334,8 +336,6 @@ OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload) m_fileReader.Read(0, m_data.data(), sz); } -void OSMElementCacheReader::LoadOffsets() { m_offsets.ReadAll(); } - // OSMElementCacheWriter --------------------------------------------------------------------------- OSMElementCacheWriter::OSMElementCacheWriter(string const & name, bool preload) : m_fileWriter(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload) @@ -345,32 +345,24 @@ OSMElementCacheWriter::OSMElementCacheWriter(string const & name, bool preload) void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); } // IntermediateDataReader -IntermediateDataReader::IntermediateDataReader(shared_ptr nodes, - feature::GenerateInfo & info) : - m_nodes(nodes), - m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache), - m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache), - m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)), - m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) +IntermediateDataReader::IntermediateDataReader(PointStorageReaderInterface const & nodes, + feature::GenerateInfo & info, bool forceReload) + : m_nodes(nodes) + , m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache, forceReload) + , m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache, forceReload) + , m_nodeToRelations(IndexReader::GetOrCreate(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT), forceReload)) + , m_wayToRelations(IndexReader::GetOrCreate(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT), forceReload)) {} -void IntermediateDataReader::LoadIndex() -{ - m_ways.LoadOffsets(); - m_relations.LoadOffsets(); - - m_nodeToRelations.ReadAll(); - m_wayToRelations.ReadAll(); -} - // IntermediateDataWriter -IntermediateDataWriter::IntermediateDataWriter(shared_ptr nodes, - feature::GenerateInfo & info) : - m_nodes(nodes), - m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache), - m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache), - m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)), - m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) {} +IntermediateDataWriter::IntermediateDataWriter(PointStorageWriterInterface & nodes, + feature::GenerateInfo & info) + : m_nodes(nodes) + , m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache) + , m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache) + , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) + , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) +{} void IntermediateDataWriter::AddRelation(Key id, RelationElement const & e) { @@ -395,34 +387,86 @@ void IntermediateDataWriter::SaveIndex() } // Functions -shared_ptr +unique_ptr CreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, string const & name) { switch (type) { case feature::GenerateInfo::NodeStorageType::File: - return make_shared(name); + return make_unique(name); case feature::GenerateInfo::NodeStorageType::Index: - return make_shared(name); + return make_unique(name); case feature::GenerateInfo::NodeStorageType::Memory: - return make_shared(name); + return make_unique(name); } UNREACHABLE(); } -shared_ptr +unique_ptr CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, string const & name) { switch (type) { case feature::GenerateInfo::NodeStorageType::File: - return make_shared(name); + return make_unique(name); case feature::GenerateInfo::NodeStorageType::Index: - return make_shared(name); + return make_unique(name); case feature::GenerateInfo::NodeStorageType::Memory: - return make_shared(name); + return make_unique(name); } UNREACHABLE(); } + +// static +std::mutex PointStorageReader::m_mutex; +// static +std::unordered_map> PointStorageReader::m_readers; + +// static +PointStorageReaderInterface const & +PointStorageReader::GetOrCreate(feature::GenerateInfo::NodeStorageType type, string const & name, + bool forceReload) +{ + string const k = to_string(static_cast(type)) + name; + std::lock_guard lock(m_mutex); + if (forceReload || m_readers.count(k) == 0) + m_readers[k] = CreatePointStorageReader(type, name); + + return *m_readers[k]; +} + +// static +std::mutex IndexReader::m_mutex; +// static +std::unordered_map IndexReader::m_indexes; + +// static +IndexFileReader const & IndexReader::GetOrCreate(std::string const & name, bool forceReload) +{ + std::lock_guard lock(m_mutex); + if (forceReload || m_indexes.count(name) == 0) + m_indexes[name] = IndexFileReader(name); + + return m_indexes[name]; +} + +IntermediateData::IntermediateData(feature::GenerateInfo & info, bool forceReload) + : m_info(info) +{ + auto const & pointReader = PointStorageReader::GetOrCreate( + info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE), + forceReload); + m_reader = make_shared(pointReader, info, forceReload); +} + +shared_ptr const & IntermediateData::GetCache() const +{ + return m_reader; +} + +std::shared_ptr IntermediateData::Clone() const +{ + return make_shared(m_info); +} } // namespace cache } // namespace generator diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index 9ec53e0f03..ecffa3433c 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -74,9 +75,9 @@ class IndexFileReader public: using Value = uint64_t; + IndexFileReader() = default; explicit IndexFileReader(std::string const & name); - void ReadAll(); bool GetValueByKey(Key key, Value & value) const; template @@ -104,7 +105,6 @@ private: }; std::vector m_elements; - FileReader m_fileReader; }; @@ -128,7 +128,8 @@ private: class OSMElementCacheReader { public: - explicit OSMElementCacheReader(std::string const & name, bool preload = false); + explicit OSMElementCacheReader(std::string const & name, bool preload = false, + bool forceReload = false); template bool Read(Key id, Value & value) @@ -157,11 +158,9 @@ public: return true; } - void LoadOffsets(); - protected: FileReader m_fileReader; - IndexFileReader m_offsets; + IndexFileReader const & m_offsets; std::string m_name; std::vector m_data; bool m_preload = false; @@ -200,13 +199,12 @@ protected: class IntermediateDataReader { public: - IntermediateDataReader(std::shared_ptr nodes, - feature::GenerateInfo & info); + IntermediateDataReader(PointStorageReaderInterface const & nodes, + feature::GenerateInfo & info, bool forceReload = false); // TODO |GetNode()|, |lat|, |lon| are used as y, x in real. - bool GetNode(Key id, double & lat, double & lon) const { return m_nodes->GetPoint(id, lat, lon); } + bool GetNode(Key id, double & lat, double & lon) const { return m_nodes.GetPoint(id, lat, lon); } bool GetWay(Key id, WayElement & e) { return m_ways.Read(id, e); } - void LoadIndex(); template void ForEachRelationByWay(Key id, ToDo && toDo) @@ -266,19 +264,19 @@ private: base::ControlFlow operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } }; - std::shared_ptr m_nodes; + PointStorageReaderInterface const & m_nodes; cache::OSMElementCacheReader m_ways; cache::OSMElementCacheReader m_relations; - cache::IndexFileReader m_nodeToRelations; - cache::IndexFileReader m_wayToRelations; + cache::IndexFileReader const & m_nodeToRelations; + cache::IndexFileReader const & m_wayToRelations; }; class IntermediateDataWriter { public: - IntermediateDataWriter(std::shared_ptr nodes, feature::GenerateInfo & info); + IntermediateDataWriter(PointStorageWriterInterface & nodes, feature::GenerateInfo & info); - void AddNode(Key id, double lat, double lon) { m_nodes->AddPoint(id, lat, lon); } + void AddNode(Key id, double lat, double lon) { m_nodes.AddPoint(id, lat, lon); } void AddWay(Key id, WayElement const & e) { m_ways.Write(id, e); } void AddRelation(Key id, RelationElement const & e); @@ -298,17 +296,54 @@ private: index.Add(v.first, relationId); } - std::shared_ptr m_nodes; + PointStorageWriterInterface & m_nodes; cache::OSMElementCacheWriter m_ways; cache::OSMElementCacheWriter m_relations; cache::IndexFileWriter m_nodeToRelations; cache::IndexFileWriter m_wayToRelations; }; -std::shared_ptr +std::unique_ptr CreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, std::string const & name); -std::shared_ptr +std::unique_ptr CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::string const & name); + +class PointStorageReader +{ +public: + static PointStorageReaderInterface const & + GetOrCreate(feature::GenerateInfo::NodeStorageType type, std::string const & name, + bool forceReload = false); + +private: + static std::mutex m_mutex; + static std::unordered_map> m_readers; +}; + +class IndexReader +{ +public: + static cache::IndexFileReader const & + GetOrCreate(std::string const & name, bool forceReload = false); + +private: + static std::mutex m_mutex; + static std::unordered_map m_indexes; +}; + +class IntermediateData +{ +public: + IntermediateData(feature::GenerateInfo & info, bool forceReload = false); + std::shared_ptr const & GetCache() const; + std::shared_ptr Clone() const; + +private: + feature::GenerateInfo & m_info; + std::shared_ptr m_reader; + + DISALLOW_COPY(IntermediateData); +}; } // namespace cache } // namespace generator