From 08474d499f2c43b325352c7e40398200ab459033 Mon Sep 17 00:00:00 2001
From: Anatoly Serdtcev <serdtcev@maps.me>
Date: Tue, 5 Nov 2019 14:09:25 +0300
Subject: [PATCH] [generator] Optimize features generatrion: async load
 intermediate data

---
 generator/intermediate_data.cpp | 22 ++++++++++++++--------
 generator/intermediate_data.hpp | 22 ++++++----------------
 2 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp
index 87f570f..43f0406 100644
--- a/generator/intermediate_data.cpp
+++ b/generator/intermediate_data.cpp
@@ -1,6 +1,9 @@
 #include "generator/intermediate_data.hpp"
 
+#include "platform/platform.hpp"
+
 #include <atomic>
+#include <future>
 #include <new>
 #include <set>
 #include <string>
@@ -391,18 +394,21 @@ void IndexFileWriter::Add(Key k, Value const & v)
 }
 
 // OSMElementCacheReader ---------------------------------------------------------------------------
-OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload, bool forceReload)
-  : m_fileReader(name)
-  , m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
+OSMElementCacheReader::OSMElementCacheReader(string const & name, bool /* preload */, bool forceReload)
+  : m_offsetsReader(GetOrCreateIndexReader(name + OFFSET_EXT, forceReload))
   , m_name(name)
-  , m_preload(preload)
 {
-  if (!m_preload)
+  if (!Platform::IsFileExistsByFullPath(name) || !boost::filesystem::file_size(name))
     return;
 
-  size_t sz = m_fileReader.Size();
-  m_data.resize(sz);
-  m_fileReader.Read(0, m_data.data(), sz);
+  m_fileMap.open(name);
+  if (!m_fileMap.is_open())
+    MYTHROW(Writer::OpenException, ("Failed to open", name));
+
+  auto readaheadTask = std::thread([data = m_fileMap.data(), size = m_fileMap.size()] {
+    ::madvise(const_cast<char*>(data), size, MADV_WILLNEED);
+  });
+  readaheadTask.detach();
 }
 
 // OSMElementCacheWriter ---------------------------------------------------------------------------
diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp
index 9f4f984..ddd084a 100644
--- a/generator/intermediate_data.hpp
+++ b/generator/intermediate_data.hpp
@@ -25,6 +25,8 @@
 #include <utility>
 #include <vector>
 
+#include <boost/iostreams/device/mapped_file.hpp>
+
 #include "defines.hpp"
 
 // Classes for reading and writing any data in file with map of offsets for
@@ -145,29 +147,17 @@ public:
       return false;
     }
 
-    uint32_t valueSize = m_preload ? *(reinterpret_cast<uint32_t *>(m_data.data() + pos)) : 0;
-    size_t offset = pos + sizeof(uint32_t);
-
-    if (!m_preload)
-    {
-      // in case not-in-memory work we read buffer
-      m_fileReader.Read(pos, &valueSize, sizeof(valueSize));
-      m_data.resize(valueSize);
-      m_fileReader.Read(pos + sizeof(valueSize), m_data.data(), valueSize);
-      offset = 0;
-    }
-
-    MemReader reader(m_data.data() + offset, valueSize);
+    uint32_t const valueSize = *(reinterpret_cast<uint32_t const *>(m_fileMap.data() + pos));
+    size_t const valueOffset = pos + sizeof(uint32_t);
+    MemReader reader(m_fileMap.data() + valueOffset, valueSize);
     value.Read(reader);
     return true;
   }
 
 protected:
-  FileReader m_fileReader;
+  boost::iostreams::mapped_file_source m_fileMap;
   IndexFileReader const & m_offsetsReader;
   std::string m_name;
-  std::vector<uint8_t> m_data;
-  bool m_preload = false;
 };
 
 class OSMElementCacheWriter