organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Optimize features generatrion: async load intermediate data

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-11-05 14:09:25 +03:00 • committed by Sergey Yershov
parent 214ec3520b
commit 08474d499f
2 changed files with 20 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
generator/intermediate_data.cpp
View file

@ -1,6 +1,9 @@
#include "generator/intermediate_data.hpp"
#include "platform/platform.hpp"
#include <atomic>
#include <future>
#include <new>
#include <set>
#include <string>
@ -391,18 +394,21 @@ void IndexFileWriter::Add(Key k, Value const & v)
}
// OSMElementCacheReader ---------------------------------------------------------------------------
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload, bool forceReload)
: m_fileReader(name)
, m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool /* preload */, bool forceReload)
: m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
, m_name(name)
, m_preload(preload)
{
if (!m_preload)
if (!Platform::IsFileExistsByFullPath(name) || !boost::filesystem::file_size(name))
return;
size_t sz = m_fileReader.Size();
m_data.resize(sz);
m_fileReader.Read(0, m_data.data(), sz);
m_fileMap.open(name);
if (!m_fileMap.is_open())
MYTHROW(Writer::OpenException, ("Failed to open", name));
auto readaheadTask = std::thread([data = m_fileMap.data(), size = m_fileMap.size()] {
::madvise(const_cast<char*>(data), size, MADV_WILLNEED);
});
readaheadTask.detach();
}
// OSMElementCacheWriter ---------------------------------------------------------------------------

22
generator/intermediate_data.hpp
View file

@ -25,6 +25,8 @@
#include <utility>
#include <vector>
#include <boost/iostreams/device/mapped_file.hpp>
#include "defines.hpp"
// Classes for reading and writing any data in file with map of offsets for
@ -145,29 +147,17 @@ public:
return false;
}
uint32_t valueSize = m_preload ? *(reinterpret_cast<uint32_t *>(m_data.data() + pos)) : 0;
size_t offset = pos + sizeof(uint32_t);
if (!m_preload)
{
// in case not-in-memory work we read buffer
m_fileReader.Read(pos, &valueSize, sizeof(valueSize));
m_data.resize(valueSize);
m_fileReader.Read(pos + sizeof(valueSize), m_data.data(), valueSize);
offset = 0;
}
MemReader reader(m_data.data() + offset, valueSize);
uint32_t const valueSize = *(reinterpret_cast<uint32_t const *>(m_fileMap.data() + pos));
size_t const valueOffset = pos + sizeof(uint32_t);
MemReader reader(m_fileMap.data() + valueOffset, valueSize);
value.Read(reader);
return true;
}
protected:
FileReader m_fileReader;
boost::iostreams::mapped_file_source m_fileMap;
IndexFileReader const & m_offsetsReader;
std::string m_name;
std::vector<uint8_t> m_data;
bool m_preload = false;
};
class OSMElementCacheWriter