From 9e693d7901e47954474cb1e92a48e1e50ef075e0 Mon Sep 17 00:00:00 2001
From: Sergey Yershov <yershov@corp.mail.ru>
Date: Fri, 23 Jan 2015 17:00:18 +0300
Subject: [PATCH] Add buffered features write

---
 generator/feature_generator.cpp | 66 +++++++++++++++++++++++++++++----
 generator/feature_generator.hpp | 12 +++++-
 generator/feature_sorter.cpp    |  2 +-
 generator/xml_element.cpp       |  2 +-
 4 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp
index 9d3004b9b9..10946a7fdd 100644
--- a/generator/feature_generator.cpp
+++ b/generator/feature_generator.cpp
@@ -125,6 +125,14 @@ public:
 FeaturesCollector::FeaturesCollector(string const & fName)
 : m_datFile(fName)
 {
+  CHECK_EQUAL(GetFileSize(m_datFile), 0, ());
+}
+
+FeaturesCollector::~FeaturesCollector()
+{
+  FlushBuffer();
+  /// Check file size
+  (void)GetFileSize(m_datFile);
 }
 
 uint32_t FeaturesCollector::GetFileSize(FileWriter const & f)
@@ -137,25 +145,69 @@ uint32_t FeaturesCollector::GetFileSize(FileWriter const & f)
   return ret;
 }
 
+template <typename ValueT, size_t ValueSizeT = sizeof(ValueT)+1>
+pair<char[ValueSizeT], uint8_t> PackValue(ValueT v)
+{
+  static_assert(is_integral<ValueT>::value, "Non integral value");
+  static_assert(is_unsigned<ValueT>::value, "Non unsigned value");
+
+  pair<char[ValueSizeT], uint8_t> res;
+  res.second = 0;
+  while (v > 127)
+  {
+    res.first[res.second++] = static_cast<uint8_t>((v & 127) | 128);
+    v >>= 7;
+  }
+  res.first[res.second++] = static_cast<uint8_t>(v);
+  return res;
+}
+
+void FeaturesCollector::FlushBuffer()
+{
+  m_datFile.Write(m_writeBuffer, m_writePosition);
+  m_baseOffset += m_writePosition;
+  m_writePosition = 0;
+}
+
+void FeaturesCollector::Flush()
+{
+  FlushBuffer();
+  m_datFile.Flush();
+}
+
+void FeaturesCollector::Write(char const *src, size_t size)
+{
+  do
+  {
+    if (m_writePosition == sizeof(m_writeBuffer))
+      FlushBuffer();
+    size_t const part_size = min(size, sizeof(m_writeBuffer) - m_writePosition);
+    memcpy(&m_writeBuffer[m_writePosition], src, part_size);
+    m_writePosition += part_size;
+    size -= part_size;
+    src += part_size;
+  } while(size > 0);
+}
+
+
 uint32_t FeaturesCollector::WriteFeatureBase(vector<char> const & bytes, FeatureBuilder1 const & fb)
 {
   size_t const sz = bytes.size();
   CHECK(sz != 0, ("Empty feature not allowed here!"));
 
-  uint32_t const offset = GetFileSize(m_datFile);
+  size_t const offset = m_baseOffset + m_writePosition;
 
-  WriteVarUint(m_datFile, sz);
-  m_datFile.Write(&bytes[0], sz);
+  auto const & packedSize = PackValue(sz);
+  Write(packedSize.first, packedSize.second);
+  Write(&bytes[0], sz);
 
   m_bounds.Add(fb.GetLimitRect());
-  return offset;
+  CHECK_EQUAL(offset, static_cast<uint32_t>(offset), ());
+  return static_cast<uint32_t>(offset);
 }
 
 void FeaturesCollector::operator() (FeatureBuilder1 const & fb)
 {
-  // Just to ensure that file size is less than 4Gb.
-  (void)GetFileSize(m_datFile);
-
   FeatureBuilder1::buffer_t bytes;
   fb.Serialize(bytes);
   (void)WriteFeatureBase(bytes, fb);
diff --git a/generator/feature_generator.hpp b/generator/feature_generator.hpp
index a607e9c9e0..ecc4f1ebf2 100644
--- a/generator/feature_generator.hpp
+++ b/generator/feature_generator.hpp
@@ -19,19 +19,29 @@ namespace feature
   // Writes features to dat file.
   class FeaturesCollector
   {
+    char m_writeBuffer[48000];
+    size_t m_writePosition = 0;
+    uint32_t m_baseOffset = 0;
+
   protected:
     FileWriter m_datFile;
-
     m2::RectD m_bounds;
 
+  private:
+    void Write(char const * src, size_t size);
+    void FlushBuffer();
+
   protected:
     static uint32_t GetFileSize(FileWriter const & f);
 
     /// @return feature offset in the file, which is used as an ID later
     uint32_t WriteFeatureBase(vector<char> const & bytes, FeatureBuilder1 const & fb);
 
+    void Flush();
+
   public:
     FeaturesCollector(string const & fName);
+    ~FeaturesCollector();
 
     void operator() (FeatureBuilder1 const & f);
   };
diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp
index 880b7884ff..9941ba28fb 100644
--- a/generator/feature_sorter.cpp
+++ b/generator/feature_sorter.cpp
@@ -128,7 +128,7 @@ namespace feature
       }
 
       // assume like we close files
-      m_datFile.Flush();
+      Flush();
 
       m_writer.Write(m_datFile.GetName(), DATA_FILE_TAG);
 
diff --git a/generator/xml_element.cpp b/generator/xml_element.cpp
index 1d7b4323d8..6c792544eb 100644
--- a/generator/xml_element.cpp
+++ b/generator/xml_element.cpp
@@ -27,7 +27,7 @@ void BaseOSMParser::AddAttr(string const & key, string const & value)
     else if (key == "lon")
       CHECK ( strings::to_double(value, m_current->lng), ("Bad node lon : ", value) );
     else if (key == "lat")
-      CHECK ( strings::to_double(value, m_current->lat), ("Bad node lon : ", value) );
+      CHECK ( strings::to_double(value, m_current->lat), ("Bad node lat : ", value) );
     else if (key == "ref")
       CHECK ( strings::to_uint64(value, m_current->ref), ("Bad node ref in way : ", value) );
     else if (key == "k")