From 4ca9859adfd3ffa46d2e33ae662302e41c35792f Mon Sep 17 00:00:00 2001 From: Sergey Yershov Date: Tue, 25 Aug 2015 13:40:53 +0300 Subject: [PATCH] Move all classes for work with intermediate cache to imtermediate_data.hpp --- generator/feature_generator.cpp | 8 +- generator/intermediate_data.hpp | 216 ++++++++++++++++- generator/osm_source.cpp | 15 +- generator/point_storage.hpp | 217 ------------------ .../generator.xcodeproj/project.pbxproj | 2 - 5 files changed, 225 insertions(+), 233 deletions(-) delete mode 100644 generator/point_storage.hpp diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp index 385ea218a0..a6dee9c6ab 100644 --- a/generator/feature_generator.cpp +++ b/generator/feature_generator.cpp @@ -1,13 +1,13 @@ #include "generator/feature_generator.hpp" -#include "generator/data_cache_file.hpp" +#include "generator/intermediate_data.hpp" #include "generator/osm_element.hpp" -#include "generator/osm_decl.hpp" #include "generator/generate_info.hpp" +#include "generator/osm_decl.hpp" +#include "indexer/cell_id.hpp" #include "indexer/data_header.hpp" #include "indexer/mercator.hpp" -#include "indexer/cell_id.hpp" #include "coding/varint.hpp" @@ -16,8 +16,8 @@ #include "base/stl_add.hpp" #include "std/bind.hpp" -#include "std/unordered_map.hpp" #include "std/target_os.hpp" +#include "std/unordered_map.hpp" /////////////////////////////////////////////////////////////////////////////////////////////////// // FeaturesCollector implementation diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index 585cf2ab7c..ad8c9735bb 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -3,6 +3,9 @@ #include "generator/osm_decl.hpp" #include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" +#include "coding/mmap_reader.hpp" #include "base/logging.hpp" @@ -126,7 +129,6 @@ public: }; } // namespace detail - template class OSMElementCache { @@ -212,4 +214,214 @@ public: inline void SaveOffsets() { m_offsets.WriteAll(); } inline void LoadOffsets() { m_offsets.ReadAll(); } }; -} // namespace cache + +/// Used to store all world nodes inside temporary index file. +/// To find node by id, just calculate offset inside index file: +/// offset_in_file = sizeof(LatLon) * node_ID +class PointStorage +{ + size_t m_processedPoint = 0; + +public: + struct LatLon + { + int32_t lat; + int32_t lon; + }; + static_assert(sizeof(LatLon) == 8, "Invalid structure size"); + + struct LatLonPos + { + uint64_t pos; + int32_t lat; + int32_t lon; + }; + static_assert(sizeof(LatLonPos) == 16, "Invalid structure size"); + + inline size_t GetProcessedPoint() const { return m_processedPoint; } + inline void IncProcessedPoint() { ++m_processedPoint; } +}; + +template +class RawFilePointStorage : public PointStorage +{ +#ifdef OMIM_OS_WINDOWS + using TFileReader = FileReader; +#else + using TFileReader = MmapReader; +#endif + + typename conditional::type m_file; + + constexpr static double const kValueOrder = 1E+7; + +public: + RawFilePointStorage(string const & name) : m_file(name) {} + + template + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLon ll; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + + m_file.Seek(id * sizeof(ll)); + m_file.Write(&ll, sizeof(ll)); + + IncProcessedPoint(); + } + + template + typename enable_if::type GetPoint(uint64_t id, double & lat, + double & lng) const + { + LatLon ll; + m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); + + // assume that valid coordinate is not (0, 0) + if (ll.lat != 0.0 || ll.lon != 0.0) + { + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; + return true; + } + LOG(LERROR, ("Node with id = ", id, " not found!")); + return false; + } +}; + +template +class RawMemPointStorage : public PointStorage +{ + typename conditional::type m_file; + + constexpr static double const kValueOrder = 1E+7; + + vector m_data; + +public: + RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF) + { + InitStorage(); + } + + ~RawMemPointStorage() { DoneStorage(); } + + template + typename enable_if::type InitStorage() {} + + template + typename enable_if::type InitStorage() + { + m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); + } + + template + typename enable_if::type DoneStorage() + { + m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon)); + } + + template + typename enable_if::type DoneStorage() {} + + template + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLon & ll = m_data[id]; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + + IncProcessedPoint(); + } + + template + typename enable_if::type GetPoint(uint64_t id, double & lat, + double & lng) const + { + LatLon const & ll = m_data[id]; + // assume that valid coordinate is not (0, 0) + if (ll.lat != 0.0 || ll.lon != 0.0) + { + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; + return true; + } + LOG(LERROR, ("Node with id = ", id, " not found!")); + return false; + } +}; + +template +class MapFilePointStorage : public PointStorage +{ + typename conditional::type m_file; + unordered_map> m_map; + + constexpr static double const kValueOrder = 1E+7; + +public: + MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage(); } + + template + typename enable_if::type InitStorage() {} + + template + typename enable_if::type InitStorage() + { + LOG(LINFO, ("Nodes reading is started")); + + uint64_t const count = m_file.Size(); + + uint64_t pos = 0; + while (pos < count) + { + LatLonPos ll; + m_file.Read(pos, &ll, sizeof(ll)); + + m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); + + pos += sizeof(ll); + } + + LOG(LINFO, ("Nodes reading is finished")); + } + + void AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLonPos ll; + ll.pos = id; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + m_file.Write(&ll, sizeof(ll)); + + IncProcessedPoint(); + } + + bool GetPoint(uint64_t id, double & lat, double & lng) const + { + auto i = m_map.find(id); + if (i == m_map.end()) + return false; + lat = static_cast(i->second.first) / kValueOrder; + lng = static_cast(i->second.second) / kValueOrder; + return true; + } +}; + +} // namespace cache diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index d57abebac6..337f1ea68a 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -1,11 +1,10 @@ #include "generator/coastlines_generator.hpp" -#include "generator/data_cache_file.hpp" #include "generator/feature_generator.hpp" +#include "generator/intermediate_data.hpp" #include "generator/osm_decl.hpp" #include "generator/osm_element.hpp" #include "generator/osm_o5m_source.hpp" #include "generator/osm_source.hpp" -#include "generator/point_storage.hpp" #include "generator/polygonizer.hpp" #include "generator/world_map_generator.hpp" #include "generator/xml_element.hpp" @@ -556,11 +555,11 @@ bool GenerateFeatures(feature::GenerateInfo & info) switch (info.m_nodeStorageType) { case feature::GenerateInfo::NodeStorageType::File: - return GenerateFeaturesImpl>(info); + return GenerateFeaturesImpl>(info); case feature::GenerateInfo::NodeStorageType::Index: - return GenerateFeaturesImpl>(info); + return GenerateFeaturesImpl>(info); case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateFeaturesImpl>(info); + return GenerateFeaturesImpl>(info); } return false; } @@ -570,11 +569,11 @@ bool GenerateIntermediateData(feature::GenerateInfo & info) switch (info.m_nodeStorageType) { case feature::GenerateInfo::NodeStorageType::File: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl>(info); case feature::GenerateInfo::NodeStorageType::Index: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl>(info); case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateIntermediateDataImpl>(info); + return GenerateIntermediateDataImpl>(info); } return false; } diff --git a/generator/point_storage.hpp b/generator/point_storage.hpp deleted file mode 100644 index 218d09caf2..0000000000 --- a/generator/point_storage.hpp +++ /dev/null @@ -1,217 +0,0 @@ -#pragma once - -#include "coding/mmap_reader.hpp" - -#include "std/iostream.hpp" -#include "std/type_traits.hpp" - -/// Used to store all world nodes inside temporary index file. -/// To find node by id, just calculate offset inside index file: -/// offset_in_file = sizeof(LatLon) * node_ID -struct LatLon -{ - int32_t lat; - int32_t lon; -}; -static_assert(sizeof(LatLon) == 8, "Invalid structure size"); - -struct LatLonPos -{ - uint64_t pos; - int32_t lat; - int32_t lon; -}; -static_assert(sizeof(LatLonPos) == 16, "Invalid structure size"); - -class BasePointStorage -{ - size_t m_processedPoint = 0; - -public: - enum class EMode { Read = false, Write = true }; - - inline size_t GetProcessedPoint() const { return m_processedPoint; } - inline void IncProcessedPoint() { ++m_processedPoint; } -}; - -template -class RawFilePointStorage : public BasePointStorage -{ -#ifdef OMIM_OS_WINDOWS - using TFileReader = FileReader; -#else - using TFileReader = MmapReader; -#endif - - typename conditional::type m_file; - - constexpr static double const kValueOrder = 1E+7; - -public: - RawFilePointStorage(string const & name) : m_file(name) {} - - template - typename enable_if::type AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - LatLon ll; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - - m_file.Seek(id * sizeof(ll)); - m_file.Write(&ll, sizeof(ll)); - - IncProcessedPoint(); - } - - template - typename enable_if::type GetPoint(uint64_t id, double & lat, - double & lng) const - { - LatLon ll; - m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); - - // assume that valid coordinate is not (0, 0) - if (ll.lat != 0.0 || ll.lon != 0.0) - { - lat = static_cast(ll.lat) / kValueOrder; - lng = static_cast(ll.lon) / kValueOrder; - return true; - } - LOG(LERROR, ("Node with id = ", id, " not found!")); - return false; - } -}; - -template -class RawMemPointStorage : public BasePointStorage -{ - typename conditional::type m_file; - - constexpr static double const kValueOrder = 1E+7; - - vector m_data; - -public: - RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF) - { - InitStorage(); - } - - ~RawMemPointStorage() { DoneStorage(); } - - template - typename enable_if::type InitStorage() {} - - template - typename enable_if::type InitStorage() - { - m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); - } - - template - typename enable_if::type DoneStorage() - { - m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon)); - } - - template - typename enable_if::type DoneStorage() {} - - template - typename enable_if::type AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - LatLon & ll = m_data[id]; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - - IncProcessedPoint(); - } - - template - typename enable_if::type GetPoint(uint64_t id, double & lat, - double & lng) const - { - LatLon const & ll = m_data[id]; - // assume that valid coordinate is not (0, 0) - if (ll.lat != 0.0 || ll.lon != 0.0) - { - lat = static_cast(ll.lat) / kValueOrder; - lng = static_cast(ll.lon) / kValueOrder; - return true; - } - LOG(LERROR, ("Node with id = ", id, " not found!")); - return false; - } -}; - -template -class MapFilePointStorage : public BasePointStorage -{ - typename conditional::type m_file; - unordered_map> m_map; - - constexpr static double const kValueOrder = 1E+7; - -public: - MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage(); } - - template - typename enable_if::type InitStorage() {} - - template - typename enable_if::type InitStorage() - { - LOG(LINFO, ("Nodes reading is started")); - - uint64_t const count = m_file.Size(); - - uint64_t pos = 0; - while (pos < count) - { - LatLonPos ll; - m_file.Read(pos, &ll, sizeof(ll)); - - m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); - - pos += sizeof(ll); - } - - LOG(LINFO, ("Nodes reading is finished")); - } - - void AddPoint(uint64_t id, double lat, double lng) - { - int64_t const lat64 = lat * kValueOrder; - int64_t const lng64 = lng * kValueOrder; - - LatLonPos ll; - ll.pos = id; - ll.lat = static_cast(lat64); - ll.lon = static_cast(lng64); - CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); - CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); - m_file.Write(&ll, sizeof(ll)); - - IncProcessedPoint(); - } - - bool GetPoint(uint64_t id, double & lat, double & lng) const - { - auto i = m_map.find(id); - if (i == m_map.end()) - return false; - lat = static_cast(i->second.first) / kValueOrder; - lng = static_cast(i->second.second) / kValueOrder; - return true; - } -}; diff --git a/xcode/generator/generator.xcodeproj/project.pbxproj b/xcode/generator/generator.xcodeproj/project.pbxproj index 1b443d52e3..76f3de1fbf 100644 --- a/xcode/generator/generator.xcodeproj/project.pbxproj +++ b/xcode/generator/generator.xcodeproj/project.pbxproj @@ -113,7 +113,6 @@ 6753405A1A3F2A7400A0A8C3 /* xml_element.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = xml_element.cpp; sourceTree = ""; }; 6753405B1A3F2A7400A0A8C3 /* xml_element.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = xml_element.hpp; sourceTree = ""; }; 6764B8921ADD6A3300DD8B15 /* osm_o5m_source.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = osm_o5m_source.hpp; sourceTree = ""; }; - 67EA644A1A7A4CF500872A69 /* point_storage.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = point_storage.hpp; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -180,7 +179,6 @@ 6753404C1A3F2A7400A0A8C3 /* osm2type.hpp */, 6726C1D31A4AFEF4005EEA39 /* osm2meta.cpp */, 6726C1D41A4AFEF4005EEA39 /* osm2meta.hpp */, - 67EA644A1A7A4CF500872A69 /* point_storage.hpp */, 6753404D1A3F2A7400A0A8C3 /* polygonizer.hpp */, 6753404E1A3F2A7400A0A8C3 /* routing_generator.cpp */, 6753404F1A3F2A7400A0A8C3 /* routing_generator.hpp */,