From ba86a3f0d8673f43dcfeeb05b76b17d54b1f1e3c Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Wed, 13 May 2020 14:49:37 +0300 Subject: [PATCH] [generator] Reduced memory usage of generator tool. --- .../booking_quality_check.cpp | 3 +- .../complex_generator/complex_generator.cpp | 3 +- .../camera_collector_tests.cpp | 3 +- generator/intermediate_data.cpp | 95 ++++++++++--------- generator/intermediate_data.hpp | 46 ++++++++- generator/raw_generator.cpp | 6 +- generator/raw_generator.hpp | 1 + 7 files changed, 103 insertions(+), 54 deletions(-) diff --git a/generator/booking_quality_check/booking_quality_check.cpp b/generator/booking_quality_check/booking_quality_check.cpp index e7db2ba425..9b34d0c6e3 100644 --- a/generator/booking_quality_check/booking_quality_check.cpp +++ b/generator/booking_quality_check/booking_quality_check.cpp @@ -316,7 +316,8 @@ void RunImpl(GenerateInfo & info) map features; LOG_SHORT(LINFO, ("OSM data:", FLAGS_osm)); - generator::cache::IntermediateData cacheLoader(info); + generator::cache::IntermediateDataObjectsCache objectsCache; + generator::cache::IntermediateData cacheLoader(objectsCache, info); auto translators = make_shared(); auto processor = make_shared>(dataset, features); translators->Append(CreateTranslator(TranslatorType::Country, processor, cacheLoader.GetCache(), info)); diff --git a/generator/complex_generator/complex_generator.cpp b/generator/complex_generator/complex_generator.cpp index e0e5cbe25d..f6e216fdb1 100644 --- a/generator/complex_generator/complex_generator.cpp +++ b/generator/complex_generator/complex_generator.cpp @@ -101,7 +101,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) { generator::RawGenerator rawGenerator(genInfo, threadsCount); auto processor = CreateProcessor(generator::ProcessorType::Complex, rawGenerator.GetQueue(), genInfo.m_intermediateDir, false /* haveBordersForWholeWorld */); - auto const cache = std::make_shared(genInfo); + generator::cache::IntermediateDataObjectsCache objectsCache; + auto const cache = std::make_shared(objectsCache, genInfo); auto translator = CreateTranslator(generator::TranslatorType::Complex, processor, cache, genInfo); auto finalProcessor = std::make_shared( genInfo.m_tmpDir, FLAGS_output, threadsCount); diff --git a/generator/generator_tests/camera_collector_tests.cpp b/generator/generator_tests/camera_collector_tests.cpp index faeccbd327..1b646cf22f 100644 --- a/generator/generator_tests/camera_collector_tests.cpp +++ b/generator/generator_tests/camera_collector_tests.cpp @@ -115,7 +115,8 @@ public: // Test load this data from cached file. auto collector = make_shared(genInfo.GetIntermediateFileName(CAMERAS_TO_WAYS_FILENAME)); - auto cache = make_shared(genInfo, true /* forceReload */); + generator::cache::IntermediateDataObjectsCache objectsCache; + auto cache = make_shared(objectsCache, genInfo); auto processor = CreateProcessor(ProcessorType::Noop); auto translator = make_shared(processor, cache); translator->SetCollector(collector); diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index e7dac77b77..2b746fbf75 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -256,33 +256,6 @@ private: FileWriter m_fileWriter; uint64_t m_numProcessedPoints = 0; }; - -IndexFileReader const & GetOrCreateIndexReader(string const & name, bool forceReload) -{ - static mutex m; - static unordered_map indexes; - - lock_guard lock(m); - if (forceReload || indexes.count(name) == 0) - indexes[name] = IndexFileReader(name); - - return indexes[name]; -} - -PointStorageReaderInterface const & -GetOrCreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, string const & name, - bool forceReload) -{ - static mutex m; - static unordered_map> readers; - - string const k = to_string(static_cast(type)) + name; - lock_guard lock(m); - if (forceReload || readers.count(k) == 0) - readers[k] = CreatePointStorageReader(type, name); - - return *readers[k]; -} } // namespace // IndexFileReader --------------------------------------------------------------------------------- @@ -348,9 +321,10 @@ void IndexFileWriter::Add(Key k, Value const & v) } // OSMElementCacheReader --------------------------------------------------------------------------- -OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload, bool forceReload) +OSMElementCacheReader::OSMElementCacheReader(IntermediateDataObjectsCache::AllocatedObjects & allocatedObjects, + string const & name, bool preload) : m_fileReader(name) - , m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload)) + , m_offsetsReader(allocatedObjects.GetOrCreateIndexReader(name + OFFSET_EXT)) , m_name(name) , m_preload(preload) { @@ -369,15 +343,47 @@ OSMElementCacheWriter::OSMElementCacheWriter(string const & name) void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); } +IntermediateDataObjectsCache::AllocatedObjects & +IntermediateDataObjectsCache::GetOrCreatePointStorageReader( + feature::GenerateInfo::NodeStorageType type, string const & name) +{ + static mutex m; + auto const k = to_string(static_cast(type)) + name; + lock_guard lock(m); + auto it = m_objects.find(k); + if (it == cend(m_objects)) + return m_objects.emplace(k, AllocatedObjects(CreatePointStorageReader(type, name))).first->second; + + return it->second; +} + +void IntermediateDataObjectsCache::Clear() +{ + m_objects = unordered_map(); +} + +IndexFileReader const & IntermediateDataObjectsCache::AllocatedObjects::GetOrCreateIndexReader( + string const & name) +{ + static mutex m; + lock_guard lock(m); + auto it = m_fileReaders.find(name); + if (it == cend(m_fileReaders)) + return m_fileReaders.emplace(name, IndexFileReader(name)).first->second; + + return it->second; +} + // IntermediateDataReader -IntermediateDataReader::IntermediateDataReader(PointStorageReaderInterface const & nodes, - feature::GenerateInfo const & info, bool forceReload) - : m_nodes(nodes) - , m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache, forceReload) - , m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache, forceReload) - , m_nodeToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT), forceReload)) - , m_wayToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT), forceReload)) - , m_relationToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT), forceReload)) +IntermediateDataReader::IntermediateDataReader( + IntermediateDataObjectsCache::AllocatedObjects & objs, + feature::GenerateInfo const & info) + : m_nodes(objs.GetPointStorageReader()) + , m_ways(objs, info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache) + , m_relations(objs, info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache) + , m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))) + , m_wayToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))) + , m_relationToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT))) {} // IntermediateDataWriter @@ -446,13 +452,14 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, string con UNREACHABLE(); } -IntermediateData::IntermediateData(feature::GenerateInfo const & info, bool forceReload) - : m_info(info) +IntermediateData::IntermediateData(IntermediateDataObjectsCache & objectsCache, + feature::GenerateInfo const & info) + : m_objectsCache(objectsCache) + , m_info(info) { - auto const & pointReader = GetOrCreatePointStorageReader( - info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE), - forceReload); - m_reader = make_shared(pointReader, info, forceReload); + auto & allocatedObjects = m_objectsCache.GetOrCreatePointStorageReader( + info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE)); + m_reader = make_shared(allocatedObjects, info); } shared_ptr const & IntermediateData::GetCache() const @@ -462,7 +469,7 @@ shared_ptr const & IntermediateData::GetCache() const shared_ptr IntermediateData::Clone() const { - return make_shared(m_info); + return make_shared(m_objectsCache, m_info); } } // namespace cache } // namespace generator diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index 9daa83f0a8..37fb8fea85 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -134,11 +134,42 @@ public: virtual bool Read(Key id, RelationElement & value) = 0; }; +class IntermediateDataObjectsCache +{ +public: + // It's thread-safe class (All public methods are thread-safe). + class AllocatedObjects + { + public: + explicit AllocatedObjects(std::unique_ptr storageReader) + : m_storageReader(std::move(storageReader)) + { + } + + PointStorageReaderInterface const & GetPointStorageReader() const { return *m_storageReader; } + + IndexFileReader const & GetOrCreateIndexReader(std::string const & name); + + private: + std::unique_ptr m_storageReader; + std::unordered_map m_fileReaders; + }; + + // It's thread-safe method. + AllocatedObjects & GetOrCreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, + std::string const & name); + + void Clear(); + +private: + std::unordered_map m_objects; +}; + class OSMElementCacheReader : public OSMElementCacheReaderInterface { public: - explicit OSMElementCacheReader(std::string const & name, bool preload = false, - bool forceReload = false); + explicit OSMElementCacheReader(IntermediateDataObjectsCache::AllocatedObjects & allocatedObjects, + std::string const & name, bool preload = false); // OSMElementCacheReaderInterface overrides: bool Read(Key id, WayElement & value) override { return Read<>(id, value); } @@ -227,8 +258,11 @@ public: class IntermediateDataReader : public IntermediateDataReaderInterface { public: - IntermediateDataReader(PointStorageReaderInterface const & nodes, - feature::GenerateInfo const & info, bool forceReload = false); + // Constructs IntermediateDataReader. + // objs - intermediate allocated objects. It's used for control allocated objects. + // info - information about the generation. + IntermediateDataReader(IntermediateDataObjectsCache::AllocatedObjects & objs, + feature::GenerateInfo const & info); // IntermediateDataReaderInterface overrides: // TODO |GetNode()|, |lat|, |lon| are used as y, x in real. @@ -346,11 +380,13 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::strin class IntermediateData { public: - explicit IntermediateData(feature::GenerateInfo const & info, bool forceReload = false); + explicit IntermediateData(IntermediateDataObjectsCache & objectsCache, + feature::GenerateInfo const & info); std::shared_ptr const & GetCache() const; std::shared_ptr Clone() const; private: + IntermediateDataObjectsCache & m_objectsCache; feature::GenerateInfo const & m_info; std::shared_ptr m_reader; diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index eaf670f434..ae462dcd6d 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -21,7 +21,7 @@ RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount, : m_genInfo(genInfo) , m_threadsCount(threadsCount) , m_chunkSize(chunkSize) - , m_cache(std::make_shared(genInfo)) + , m_cache(std::make_shared(m_intermediateDataObjectsCache, genInfo)) , m_queue(std::make_shared()) , m_translators(std::make_shared()) { @@ -29,7 +29,8 @@ RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount, void RawGenerator::ForceReloadCache() { - m_cache = std::make_shared(m_genInfo, true /* forceReload */); + m_intermediateDataObjectsCache.Clear(); + m_cache = std::make_shared(m_intermediateDataObjectsCache, m_genInfo); } std::shared_ptr RawGenerator::GetQueue() { return m_queue; } @@ -85,6 +86,7 @@ bool RawGenerator::Execute() m_translators.reset(); m_cache.reset(); m_queue.reset(); + m_intermediateDataObjectsCache.Clear(); while (!m_finalProcessors.empty()) { diff --git a/generator/raw_generator.hpp b/generator/raw_generator.hpp index b4cccf4544..fd4911119b 100644 --- a/generator/raw_generator.hpp +++ b/generator/raw_generator.hpp @@ -49,6 +49,7 @@ private: feature::GenerateInfo & m_genInfo; size_t m_threadsCount; size_t m_chunkSize; + cache::IntermediateDataObjectsCache m_intermediateDataObjectsCache; std::shared_ptr m_cache; std::shared_ptr m_queue; std::shared_ptr m_translators;