kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

[generator] Reduced memory usage of generator tool.

Browse source
This commit is contained in:
Maksim Andrianov 2020-05-13 14:49:37 +03:00 • committed by mpimenov
parent fe8a0e6a57
commit ba86a3f0d8
7 changed files with 103 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
generator/booking_quality_check/booking_quality_check.cpp
View file

@ -316,7 +316,8 @@ void RunImpl(GenerateInfo & info)
map<base::GeoObjectId, FeatureBuilder> features;
LOG_SHORT(LINFO, ("OSM data:", FLAGS_osm));
generator::cache::IntermediateData cacheLoader(info);
generator::cache::IntermediateDataObjectsCache objectsCache;
generator::cache::IntermediateData cacheLoader(objectsCache, info);
auto translators = make_shared<TranslatorCollection>();
auto processor = make_shared<ProcessorBooking<Dataset>>(dataset, features);
translators->Append(CreateTranslator(TranslatorType::Country, processor, cacheLoader.GetCache(), info));

3
generator/complex_generator/complex_generator.cpp
View file

@ -101,7 +101,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) {
generator::RawGenerator rawGenerator(genInfo, threadsCount);
auto processor = CreateProcessor(generator::ProcessorType::Complex, rawGenerator.GetQueue(),
genInfo.m_intermediateDir, false /* haveBordersForWholeWorld */);
auto const cache = std::make_shared<generator::cache::IntermediateData>(genInfo);
generator::cache::IntermediateDataObjectsCache objectsCache;
auto const cache = std::make_shared<generator::cache::IntermediateData>(objectsCache, genInfo);
auto translator = CreateTranslator(generator::TranslatorType::Complex, processor, cache, genInfo);
auto finalProcessor = std::make_shared<generator::ComplexFinalProcessor>(
genInfo.m_tmpDir, FLAGS_output, threadsCount);

3
generator/generator_tests/camera_collector_tests.cpp
View file

@ -115,7 +115,8 @@ public:
// Test load this data from cached file.
auto collector = make_shared<CameraCollector>(genInfo.GetIntermediateFileName(CAMERAS_TO_WAYS_FILENAME));
auto cache = make_shared<generator::cache::IntermediateData>(genInfo, true /* forceReload */);
generator::cache::IntermediateDataObjectsCache objectsCache;
auto cache = make_shared<generator::cache::IntermediateData>(objectsCache, genInfo);
auto processor = CreateProcessor(ProcessorType::Noop);
auto translator = make_shared<TranslatorForTest>(processor, cache);
translator->SetCollector(collector);

95
generator/intermediate_data.cpp
View file

@ -256,33 +256,6 @@ private:
FileWriter m_fileWriter;
uint64_t m_numProcessedPoints = 0;
};
IndexFileReader const & GetOrCreateIndexReader(string const & name, bool forceReload)
{
static mutex m;
static unordered_map<string, IndexFileReader> indexes;
lock_guard<mutex> lock(m);
if (forceReload || indexes.count(name) == 0)
indexes[name] = IndexFileReader(name);
return indexes[name];
}
PointStorageReaderInterface const &
GetOrCreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, string const & name,
bool forceReload)
{
static mutex m;
static unordered_map<string, unique_ptr<PointStorageReaderInterface>> readers;
string const k = to_string(static_cast<int>(type)) + name;
lock_guard<mutex> lock(m);
if (forceReload || readers.count(k) == 0)
readers[k] = CreatePointStorageReader(type, name);
return *readers[k];
}
} // namespace
// IndexFileReader ---------------------------------------------------------------------------------
@ -348,9 +321,10 @@ void IndexFileWriter::Add(Key k, Value const & v)
}
// OSMElementCacheReader ---------------------------------------------------------------------------
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload, bool forceReload)
OSMElementCacheReader::OSMElementCacheReader(IntermediateDataObjectsCache::AllocatedObjects & allocatedObjects,
string const & name, bool preload)
: m_fileReader(name)
, m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
, m_offsetsReader(allocatedObjects.GetOrCreateIndexReader(name + OFFSET_EXT))
, m_name(name)
, m_preload(preload)
{
@ -369,15 +343,47 @@ OSMElementCacheWriter::OSMElementCacheWriter(string const & name)
void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); }
IntermediateDataObjectsCache::AllocatedObjects &
IntermediateDataObjectsCache::GetOrCreatePointStorageReader(
feature::GenerateInfo::NodeStorageType type, string const & name)
{
static mutex m;
auto const k = to_string(static_cast<int>(type)) + name;
lock_guard<mutex> lock(m);
auto it = m_objects.find(k);
if (it == cend(m_objects))
return m_objects.emplace(k, AllocatedObjects(CreatePointStorageReader(type, name))).first->second;
return it->second;
}
void IntermediateDataObjectsCache::Clear()
{
m_objects = unordered_map<string, AllocatedObjects>();
}
IndexFileReader const & IntermediateDataObjectsCache::AllocatedObjects::GetOrCreateIndexReader(
string const & name)
{
static mutex m;
lock_guard<mutex> lock(m);
auto it = m_fileReaders.find(name);
if (it == cend(m_fileReaders))
return m_fileReaders.emplace(name, IndexFileReader(name)).first->second;
return it->second;
}
// IntermediateDataReader
IntermediateDataReader::IntermediateDataReader(PointStorageReaderInterface const & nodes,
feature::GenerateInfo const & info, bool forceReload)
: m_nodes(nodes)
, m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache, forceReload)
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache, forceReload)
, m_nodeToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT), forceReload))
, m_wayToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT), forceReload))
, m_relationToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT), forceReload))
IntermediateDataReader::IntermediateDataReader(
IntermediateDataObjectsCache::AllocatedObjects & objs,
feature::GenerateInfo const & info)
: m_nodes(objs.GetPointStorageReader())
, m_ways(objs, info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache)
, m_relations(objs, info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache)
, m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)))
, m_wayToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)))
, m_relationToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT)))
{}
// IntermediateDataWriter
@ -446,13 +452,14 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, string con
UNREACHABLE();
}
IntermediateData::IntermediateData(feature::GenerateInfo const & info, bool forceReload)
: m_info(info)
IntermediateData::IntermediateData(IntermediateDataObjectsCache & objectsCache,
feature::GenerateInfo const & info)
: m_objectsCache(objectsCache)
, m_info(info)
{
auto const & pointReader = GetOrCreatePointStorageReader(
info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE),
forceReload);
m_reader = make_shared<IntermediateDataReader>(pointReader, info, forceReload);
auto & allocatedObjects = m_objectsCache.GetOrCreatePointStorageReader(
info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE));
m_reader = make_shared<IntermediateDataReader>(allocatedObjects, info);
}
shared_ptr<IntermediateDataReader> const & IntermediateData::GetCache() const
@ -462,7 +469,7 @@ shared_ptr<IntermediateDataReader> const & IntermediateData::GetCache() const
shared_ptr<IntermediateData> IntermediateData::Clone() const
{
return make_shared<IntermediateData>(m_info);
return make_shared<IntermediateData>(m_objectsCache, m_info);
}
} // namespace cache
} // namespace generator

46
generator/intermediate_data.hpp
View file

@ -134,11 +134,42 @@ public:
virtual bool Read(Key id, RelationElement & value) = 0;
};
class IntermediateDataObjectsCache
{
public:
// It's thread-safe class (All public methods are thread-safe).
class AllocatedObjects
{
public:
explicit AllocatedObjects(std::unique_ptr<PointStorageReaderInterface> storageReader)
: m_storageReader(std::move(storageReader))
{
}
PointStorageReaderInterface const & GetPointStorageReader() const { return *m_storageReader; }
IndexFileReader const & GetOrCreateIndexReader(std::string const & name);
private:
std::unique_ptr<PointStorageReaderInterface> m_storageReader;
std::unordered_map<std::string, IndexFileReader> m_fileReaders;
};
// It's thread-safe method.
AllocatedObjects & GetOrCreatePointStorageReader(feature::GenerateInfo::NodeStorageType type,
std::string const & name);
void Clear();
private:
std::unordered_map<std::string, AllocatedObjects> m_objects;
};
class OSMElementCacheReader : public OSMElementCacheReaderInterface
{
public:
explicit OSMElementCacheReader(std::string const & name, bool preload = false,
bool forceReload = false);
explicit OSMElementCacheReader(IntermediateDataObjectsCache::AllocatedObjects & allocatedObjects,
std::string const & name, bool preload = false);
// OSMElementCacheReaderInterface overrides:
bool Read(Key id, WayElement & value) override { return Read<>(id, value); }
@ -227,8 +258,11 @@ public:
class IntermediateDataReader : public IntermediateDataReaderInterface
{
public:
IntermediateDataReader(PointStorageReaderInterface const & nodes,
feature::GenerateInfo const & info, bool forceReload = false);
// Constructs IntermediateDataReader.
// objs - intermediate allocated objects. It's used for control allocated objects.
// info - information about the generation.
IntermediateDataReader(IntermediateDataObjectsCache::AllocatedObjects & objs,
feature::GenerateInfo const & info);
// IntermediateDataReaderInterface overrides:
// TODO |GetNode()|, |lat|, |lon| are used as y, x in real.
@ -346,11 +380,13 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::strin
class IntermediateData
{
public:
explicit IntermediateData(feature::GenerateInfo const & info, bool forceReload = false);
explicit IntermediateData(IntermediateDataObjectsCache & objectsCache,
feature::GenerateInfo const & info);
std::shared_ptr<IntermediateDataReader> const & GetCache() const;
std::shared_ptr<IntermediateData> Clone() const;
private:
IntermediateDataObjectsCache & m_objectsCache;
feature::GenerateInfo const & m_info;
std::shared_ptr<IntermediateDataReader> m_reader;

6
generator/raw_generator.cpp
View file

@ -21,7 +21,7 @@ RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount,
: m_genInfo(genInfo)
, m_threadsCount(threadsCount)
, m_chunkSize(chunkSize)
, m_cache(std::make_shared<generator::cache::IntermediateData>(genInfo))
, m_cache(std::make_shared<generator::cache::IntermediateData>(m_intermediateDataObjectsCache, genInfo))
, m_queue(std::make_shared<FeatureProcessorQueue>())
, m_translators(std::make_shared<TranslatorCollection>())
{
@ -29,7 +29,8 @@ RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount,
void RawGenerator::ForceReloadCache()
{
m_cache = std::make_shared<cache::IntermediateData>(m_genInfo, true /* forceReload */);
m_intermediateDataObjectsCache.Clear();
m_cache = std::make_shared<cache::IntermediateData>(m_intermediateDataObjectsCache, m_genInfo);
}
std::shared_ptr<FeatureProcessorQueue> RawGenerator::GetQueue() { return m_queue; }
@ -85,6 +86,7 @@ bool RawGenerator::Execute()
m_translators.reset();
m_cache.reset();
m_queue.reset();
m_intermediateDataObjectsCache.Clear();
while (!m_finalProcessors.empty())
{

1
generator/raw_generator.hpp
View file

@ -49,6 +49,7 @@ private:
feature::GenerateInfo & m_genInfo;
size_t m_threadsCount;
size_t m_chunkSize;
cache::IntermediateDataObjectsCache m_intermediateDataObjectsCache;
std::shared_ptr<cache::IntermediateData> m_cache;
std::shared_ptr<FeatureProcessorQueue> m_queue;
std::shared_ptr<TranslatorCollection> m_translators;