organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Refactor features generator: delete unused preload and force-reload modes

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-11-05 14:49:48 +03:00 • committed by Sergey Yershov
parent 79ee32b690
commit 2ffce3cfde
6 changed files with 27 additions and 69 deletions

1
generator/generate_info.hpp
View file

@ -41,7 +41,6 @@ struct GenerateInfo
unsigned int m_threadsCount{1};
bool m_preloadCache = false;
bool m_verbose = false;
GenerateInfo() = default;

2
generator/generator_tests/intermediate_data_test.cpp
View file

@ -185,7 +185,7 @@ void TestIntermediateDataGeneration(
auto osmElements =
ReadOsmElements(genInfo.m_osmFileName, osmFormatParsers.at(osmFileTypeExtension));
auto const & intermediateData = cache::IntermediateData{genInfo, true /* forceReload */};
auto const & intermediateData = cache::IntermediateData{genInfo};
auto const & cache = intermediateData.GetCache();
dataTester(osmElements, *cache);
}

62
generator/intermediate_data.cpp
View file

@ -299,33 +299,6 @@ private:
std::mutex m_updateMutex;
uint64_t m_numProcessedPoints = 0;
};
IndexFileReader const & GetOrCreateIndexReader(string const & name, bool forceReload)
{
static mutex m;
static unordered_map<string, IndexFileReader> indexes;
lock_guard<mutex> lock(m);
if (forceReload || indexes.count(name) == 0)
indexes[name] = IndexFileReader(name);
return indexes[name];
}
PointStorageReaderInterface const &
GetOrCreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, string const & name,
bool forceReload)
{
static mutex m;
static unordered_map<string, unique_ptr<PointStorageReaderInterface>> readers;
string const k = to_string(static_cast<int>(type)) + name;
lock_guard<mutex> lock(m);
if (forceReload || readers.count(k) == 0)
readers[k] = CreatePointStorageReader(type, name);
return *readers[k];
}
} // namespace
// IndexFileReader ---------------------------------------------------------------------------------
@ -394,8 +367,8 @@ void IndexFileWriter::Add(Key k, Value const & v)
}
// OSMElementCacheReader ---------------------------------------------------------------------------
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool /* preload */, bool forceReload)
: m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
OSMElementCacheReader::OSMElementCacheReader(string const & name)
: m_offsetsReader(name + OFFSET_EXT)
, m_name(name)
{
if (!Platform::IsFileExistsByFullPath(name) || !boost::filesystem::file_size(name))
@ -412,33 +385,32 @@ OSMElementCacheReader::OSMElementCacheReader(string const & name, bool /* preloa
}
// OSMElementCacheWriter ---------------------------------------------------------------------------
OSMElementCacheWriter::OSMElementCacheWriter(string const & name, bool preload)
OSMElementCacheWriter::OSMElementCacheWriter(string const & name)
: m_fileWriter(name, FileWriter::OP_WRITE_TRUNCATE, 10 * 1024 * 1024 /* bufferSize */)
, m_currOffset{m_fileWriter.Pos()}
, m_offsets(name + OFFSET_EXT)
, m_name(name)
, m_preload(preload)
{
}
void OSMElementCacheWriter::SaveOffsets() { m_offsets.WriteAll(); }
// IntermediateDataReader
IntermediateDataReader::IntermediateDataReader(PointStorageReaderInterface const & nodes,
feature::GenerateInfo const & info, bool forceReload)
: m_nodes(nodes)
, m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache, forceReload)
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache, forceReload)
, m_nodeToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT), forceReload))
, m_wayToRelations(GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT), forceReload))
IntermediateDataReader::IntermediateDataReader(feature::GenerateInfo const & info)
: m_nodes{CreatePointStorageReader(info.m_nodeStorageType,
info.GetIntermediateFileName(NODES_FILE))}
, m_ways(info.GetIntermediateFileName(WAYS_FILE))
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE))
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))
{}
// IntermediateDataWriter
IntermediateDataWriter::IntermediateDataWriter(PointStorageWriterInterface & nodes,
feature::GenerateInfo const & info)
: m_nodes(nodes)
, m_ways(info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache)
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache)
, m_ways(info.GetIntermediateFileName(WAYS_FILE))
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE))
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))
{}
@ -533,14 +505,10 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, string con
UNREACHABLE();
}
IntermediateData::IntermediateData(feature::GenerateInfo const & info, bool forceReload)
IntermediateData::IntermediateData(feature::GenerateInfo const & info)
: m_info(info)
{
auto const & pointReader = GetOrCreatePointStorageReader(
info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE),
forceReload);
m_reader = make_shared<IntermediateDataReader>(pointReader, info, forceReload);
}
, m_reader{make_shared<IntermediateDataReader>(info)}
{ }
shared_ptr<IntermediateDataReader> const & IntermediateData::GetCache() const
{

25
generator/intermediate_data.hpp
View file

@ -134,8 +134,7 @@ private:
class OSMElementCacheReader
{
public:
explicit OSMElementCacheReader(std::string const & name, bool preload = false,
bool forceReload = false);
explicit OSMElementCacheReader(std::string const & name);
template <class Value>
bool Read(Key id, Value & value) const
@ -156,14 +155,14 @@ public:
protected:
boost::iostreams::mapped_file_source m_fileMap;
IndexFileReader const & m_offsetsReader;
IndexFileReader m_offsetsReader;
std::string m_name;
};
class OSMElementCacheWriter
{
public:
explicit OSMElementCacheWriter(std::string const & name, bool preload = false);
explicit OSMElementCacheWriter(std::string const & name);
template <typename Value>
void Write(Key id, Value const & value)
@ -224,7 +223,6 @@ protected:
std::mutex m_offsetsMutex;
std::string m_name;
std::vector<uint8_t> m_data;
bool m_preload = false;
private:
template <typename Value, typename Writer>
@ -249,11 +247,10 @@ private:
class IntermediateDataReader
{
public:
IntermediateDataReader(PointStorageReaderInterface const & nodes,
feature::GenerateInfo const & info, bool forceReload = false);
IntermediateDataReader(feature::GenerateInfo const & info);
// TODO |GetNode()|, |lat|, |lon| are used as y, x in real.
bool GetNode(Key id, double & lat, double & lon) const { return m_nodes.GetPoint(id, lat, lon); }
bool GetNode(Key id, double & lat, double & lon) const { return m_nodes->GetPoint(id, lat, lon); }
bool GetWay(Key id, WayElement & e) const { return m_ways.Read(id, e); }
template <typename ToDo>
@ -315,11 +312,11 @@ private:
base::ControlFlow operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); }
};
PointStorageReaderInterface const & m_nodes;
cache::OSMElementCacheReader const m_ways;
cache::OSMElementCacheReader const m_relations;
cache::IndexFileReader const & m_nodeToRelations;
cache::IndexFileReader const & m_wayToRelations;
std::unique_ptr<PointStorageReaderInterface> m_nodes;
cache::OSMElementCacheReader m_ways;
cache::OSMElementCacheReader m_relations;
cache::IndexFileReader m_nodeToRelations;
cache::IndexFileReader m_wayToRelations;
};
class IntermediateDataWriter
@ -372,7 +369,7 @@ CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::strin
class IntermediateData : public std::enable_shared_from_this<IntermediateData>
{
public:
explicit IntermediateData(feature::GenerateInfo const & info, bool forceReload = false);
explicit IntermediateData(feature::GenerateInfo const & info);
std::shared_ptr<IntermediateDataReader> const & GetCache() const;
std::shared_ptr<IntermediateData const> Clone() const;

5
generator/raw_generator.cpp
View file

@ -29,11 +29,6 @@ RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t chunkSize)
{
}
void RawGenerator::ForceReloadCache()
{
m_cache = std::make_shared<cache::IntermediateData>(m_genInfo, true /* forceReload */);
}
std::shared_ptr<FeatureProcessorQueue> RawGenerator::GetQueue()
{
return m_queue;

1
generator/raw_generator.hpp
View file

@ -35,7 +35,6 @@ public:
bool Execute();
std::vector<std::string> const & GetNames() const;
std::shared_ptr<FeatureProcessorQueue> GetQueue();
void ForceReloadCache();
private:
using FinalProcessorPtr = std::shared_ptr<FinalProcessorIntermediateMwmInterface>;