Optimization memory reallocation while reading variable length records

Simple optimization get_mark_value
Correct working with paths in generator_tool

coding/var_record_reader.hpp

@@ -31,23 +31,28 @@ public:
     ASSERT_GREATER_OR_EQUAL(expectedRecordSize, 4, ());
   }
 
-  uint64_t ReadRecord(uint64_t const pos, vector<char> & buffer, uint32_t & recordOffset) const
+  uint64_t ReadRecord(uint64_t const pos, vector<char> & buffer, uint32_t & recordOffset, uint32_t & actualSize) const
   {
     ASSERT_LESS(pos, m_ReaderSize, ());
     uint32_t const initialSize = static_cast<uint32_t>(
         min(static_cast<uint64_t>(m_ExpectedRecordSize), m_ReaderSize - pos));
-    buffer.resize(initialSize);
+
+    if (buffer.size() < initialSize)
+      buffer.resize(initialSize);
+
     m_Reader.Read(pos, &buffer[0], initialSize);
     ArrayByteSource source(&buffer[0]);
     uint32_t const recordSize = VarRecordSizeReaderFn(source);
     uint32_t const recordSizeSize = static_cast<uint32_t>(source.PtrC() - &buffer[0]);
     uint32_t const fullSize = recordSize + recordSizeSize;
     ASSERT_LESS_OR_EQUAL(pos + fullSize, m_ReaderSize, ());
-    buffer.resize(fullSize);
+    if (buffer.size() < fullSize)
+      buffer.resize(fullSize);
     if (initialSize < fullSize)
       m_Reader.Read(pos + initialSize, &buffer[initialSize], fullSize - initialSize);
 
     recordOffset = recordSizeSize;
+    actualSize = fullSize;
     return pos + fullSize;
   }
 
@@ -58,10 +63,10 @@ public:
     vector<char> buffer;
     while (pos < m_ReaderSize)
     {
-      uint32_t offset;
-      uint64_t nextPos = ReadRecord(pos, buffer, offset);
+      uint32_t offset = 0, size = 0;
+      uint64_t nextPos = ReadRecord(pos, buffer, offset, size);
       // uint64_t -> uint32_t : assume that feature dat file not more than 4Gb
-      f(static_cast<uint32_t>(pos), &buffer[offset], static_cast<uint32_t>(buffer.size() - offset));
+      f(static_cast<uint32_t>(pos), &buffer[offset], static_cast<uint32_t>(size - offset));
       pos = nextPos;
     }
     ASSERT_EQUAL(pos, m_ReaderSize, ());

generator/generator_tool/generator_tool.cpp

@@ -220,7 +220,7 @@ int main(int argc, char ** argv)
     {
       LOG(LINFO, ("Generating search index for ", datFile));
 
-      if (!indexer::BuildSearchIndexFromDatFile(country + DATA_FILE_EXTENSION, true))
+      if (!indexer::BuildSearchIndexFromDatFile(datFile, true))
         LOG(LCRITICAL, ("Error generating search index."));
     }
   }

generator/osm2type.cpp

@@ -27,15 +27,20 @@ namespace ftype
       static char const * aFalse[] = { "no", "false", "-1" };
 
       strings::SimpleTokenizer it(v, "|");
-      while (it)
+      if (it)
       {
-        if (strings::IsInArray(aTrue, *it))
-          return 1;
+        bool allowedKey = (k != "layer" && k != "oneway");
+        while (it)
+        {
+          string const &part = *it;
+          if (allowedKey && strings::IsInArray(aFalse, part))
+            return -1;
 
-        if (k != "layer" && k != "oneway" && strings::IsInArray(aFalse, *it))
-          return -1;
+          if (strings::IsInArray(aTrue, part))
+            return 1;
 
-        ++it;
+          ++it;
+        }
       }
 
       // "~" means no this tag, so sometimes it means true,
@@ -67,11 +72,12 @@ namespace ftype
         if (p->childs[i].name == "tag")
         {
           string & k = p->childs[i].attrs["k"];
-          string & v = p->childs[i].attrs["v"];
 
           if (k.empty() || is_skip_tag(k))
             continue;
 
+          string & v = p->childs[i].attrs["v"];
+
           // this means "no"
           if (get_mark_value(k, v) == -1)
             continue;

generator/osm_element.hpp

@@ -123,10 +123,10 @@ class SecondPassParser : public BaseOSMParser
 
     bool HasName() const
     {
-      for (auto p : m_current->childs)
+      for (auto const & p : m_current->childs)
       {
         if (p.name == "tag")
-          for (auto a : p.attrs)
+          for (auto const & a : p.attrs)
             if (strings::StartsWith(a.first, "name"))
               return true;
       }
@@ -151,7 +151,7 @@ class SecondPassParser : public BaseOSMParser
       bool const isBoundary = isWay && (type == "boundary") && IsAcceptBoundary(e);
       bool const hasName = HasName();
 
-      for (auto p : e.tags)
+      for (auto const &p : e.tags)
       {
         /// @todo Skip common key tags.
         if (p.first == "type" || p.first == "route")

indexer/features_vector.hpp

@@ -17,8 +17,8 @@ public:
 
   void Get(uint64_t pos, FeatureType & ft) const
   {
-    uint32_t offset;
-    m_RecordReader.ReadRecord(pos, m_buffer, offset);
+    uint32_t offset = 0, size = 0;
+    m_RecordReader.ReadRecord(pos, m_buffer, offset, size);
 
     ft.Deserialize(m_LoadInfo.GetLoader(), &m_buffer[offset]);
   }

indexer/search_index_builder.cpp

@@ -396,15 +396,15 @@ void BuildSearchIndex(FilesContainerR const & cont, CategoriesHolder const & cat
 
 }
 
-bool indexer::BuildSearchIndexFromDatFile(string const & fName, bool forceRebuild)
+bool indexer::BuildSearchIndexFromDatFile(string const & datFile, bool forceRebuild)
 {
   LOG(LINFO, ("Start building search index. Bits = ", search::POINT_CODING_BITS));
 
   try
   {
     Platform & pl = GetPlatform();
-    string const datFile = pl.WritablePathForFile(fName);
-    string const tmpFile = pl.WritablePathForFile(fName + ".search_index_2.tmp");
+    string const tmpFile1 = datFile + ".search_index_1.tmp";
+    string const tmpFile2 = datFile + ".search_index_2.tmp";
 
     {
       FilesContainerR readCont(datFile);
@@ -412,12 +412,11 @@ bool indexer::BuildSearchIndexFromDatFile(string const & fName, bool forceRebuil
       if (!forceRebuild && readCont.IsExist(SEARCH_INDEX_FILE_TAG))
         return true;
 
-      FileWriter writer(tmpFile);
+      FileWriter writer(tmpFile2);
 
       CategoriesHolder catHolder(pl.GetReader(SEARCH_CATEGORIES_FILE_NAME));
 
-      BuildSearchIndex(readCont, catHolder, writer,
-                       pl.WritablePathForFile(fName + ".search_index_1.tmp"));
+      BuildSearchIndex(readCont, catHolder, writer, tmpFile1);
 
       LOG(LINFO, ("Search index size = ", writer.Size()));
     }
@@ -426,10 +425,10 @@ bool indexer::BuildSearchIndexFromDatFile(string const & fName, bool forceRebuil
       // Write to container in reversed order.
       FilesContainerW writeCont(datFile, FileWriter::OP_WRITE_EXISTING);
       FileWriter writer = writeCont.GetWriter(SEARCH_INDEX_FILE_TAG);
-      rw_ops::Reverse(FileReader(tmpFile), writer);
+      rw_ops::Reverse(FileReader(tmpFile2), writer);
     }
 
-    FileWriter::DeleteFileX(tmpFile);
+    FileWriter::DeleteFileX(tmpFile2);
   }
   catch (Reader::Exception const & e)
   {