[indexer] Optimize centers table to use less CoordBits.

generator/centers_table_builder.cpp

@@ -52,7 +52,7 @@ bool BuildCentersTableFromDataFile(std::string const & filename, bool forceRebui
       feature::DataHeader const header(rcont);
       FeaturesVector const features(rcont, header, table.get());
 
-      builder.SetGeometryCodingParams(header.GetDefGeometryCodingParams());
+      builder.SetGeometryParams(header.GetBounds());
       features.ForEach([&](FeatureType & ft, uint32_t featureId) {
         builder.Put(featureId, feature::GetCenter(ft));
       });

generator/search_index_builder.cpp

@@ -727,7 +727,8 @@ bool BuildPostcodesImpl(FilesContainerR & container, storage::CountryId const &
 
   {
     search::CentersTableBuilder builder;
-    builder.SetGeometryCodingParams(feature::DataHeader(container).GetDefGeometryCodingParams());
+
+    builder.SetGeometryParams(feature::DataHeader(container).GetBounds());
     for (size_t i = 0; i < valueMapping.size(); ++i)
       builder.Put(base::asserted_cast<uint32_t>(i), valueMapping[i]);
 

indexer/centers_table.cpp

@@ -26,30 +26,80 @@ using namespace std;
 
 namespace search
 {
+void CentersTable::Header::Read(Reader & reader)
+{
+  NonOwningReaderSource source(reader);
+  m_version = static_cast<Version>(ReadPrimitiveFromSource<uint8_t>(source));
+  CHECK_EQUAL(static_cast<uint8_t>(m_version), static_cast<uint8_t>(Version::V1), ());
+  m_geometryParamsOffset = ReadPrimitiveFromSource<uint32_t>(source);
+  m_geometryParamsSize = ReadPrimitiveFromSource<uint32_t>(source);
+  m_centersOffset = ReadPrimitiveFromSource<uint32_t>(source);
+  m_centersSize = ReadPrimitiveFromSource<uint32_t>(source);
+}
+
 bool CentersTable::Get(uint32_t id, m2::PointD & center)
 {
   m2::PointU pointu;
   if (!m_map->Get(id, pointu))
     return false;
 
-  center = PointUToPointD(pointu, m_codingParams.GetCoordBits());
+  if (m_version == Version::V0)
+    center = PointUToPointD(pointu, m_codingParams.GetCoordBits());
+  else if (m_version == Version::V1)
+    center = PointUToPointD(pointu, m_codingParams.GetCoordBits(), m_limitRect);
+  else
+    CHECK(false, ("Unknown CentersTable format."));
+
   return true;
 }
 
 // CentersTable ------------------------------------------------------------------------------------
 // static
-unique_ptr<CentersTable> CentersTable::Load(Reader & reader,
-                                            serial::GeometryCodingParams const & codingParams)
+unique_ptr<CentersTable> CentersTable::LoadV0(Reader & reader,
+                                              serial::GeometryCodingParams const & codingParams)
 {
   auto table = make_unique<CentersTable>();
-  if (!table->Init(reader, codingParams))
+  table->m_version = Version::V0;
+  if (!table->Init(reader, codingParams, {} /* limitRect */))
     return {};
   return table;
 }
 
-bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & codingParams)
+unique_ptr<CentersTable> CentersTable::LoadV1(Reader & reader)
+{
+  auto table = make_unique<CentersTable>();
+  table->m_version = Version::V1;
+
+  Header header;
+  header.Read(reader);
+
+  auto geometryParamsSubreader =
+      reader.CreateSubReader(header.m_geometryParamsOffset, header.m_geometryParamsSize);
+  if (!geometryParamsSubreader)
+    return {};
+  NonOwningReaderSource geometryParamsSource(*geometryParamsSubreader);
+  serial::GeometryCodingParams codingParams;
+  codingParams.Load(geometryParamsSource);
+  auto minX = ReadPrimitiveFromSource<uint32_t>(geometryParamsSource);
+  auto minY = ReadPrimitiveFromSource<uint32_t>(geometryParamsSource);
+  auto maxX = ReadPrimitiveFromSource<uint32_t>(geometryParamsSource);
+  auto maxY = ReadPrimitiveFromSource<uint32_t>(geometryParamsSource);
+  m2::RectD limitRect(PointUToPointD({minX, minY}, kPointCoordBits),
+                      PointUToPointD({maxX, maxY}, kPointCoordBits));
+
+  table->m_centersSubreader = reader.CreateSubReader(header.m_centersOffset, header.m_centersSize);
+  if (!table->m_centersSubreader)
+    return {};
+  if (!table->Init(*(table->m_centersSubreader), codingParams, limitRect))
+    return {};
+  return table;
+}
+
+bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & codingParams,
+                        m2::RectD const & limitRect)
 {
   m_codingParams = codingParams;
+  m_limitRect = limitRect;
   // Decodes block encoded by writeBlockCallback from CentersTableBuilder::Freeze.
   auto const readBlockCallback = [&](NonOwningReaderSource & source, uint32_t blockSize,
                                      vector<m2::PointU> & values) {
@@ -69,29 +119,84 @@ bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & co
 }
 
 // CentersTableBuilder -----------------------------------------------------------------------------
+void CentersTableBuilder::SetGeometryParams(m2::RectD const & limitRect, double pointAccuracy)
+{
+  auto const coordBits = GetCoordBits(limitRect, pointAccuracy);
+  m_codingParams = serial::GeometryCodingParams(coordBits, limitRect.Center());
+  m_limitRect = limitRect;
+}
+
 void CentersTableBuilder::Put(uint32_t featureId, m2::PointD const & center)
 {
-  m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits()));
+  m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits(), m_limitRect));
+}
+
+// Each center is encoded as delta from some prediction.
+// For the first center in the block map base point is used as a prediction, for all other
+// centers in the block previous center is used as a prediction.
+void CentersTableBuilder::WriteBlock(Writer & w, vector<m2::PointU>::const_iterator begin,
+                                     vector<m2::PointU>::const_iterator end) const
+{
+  uint64_t delta = coding::EncodePointDeltaAsUint(*begin, m_codingParams.GetBasePoint());
+  WriteVarUint(w, delta);
+  auto prevIt = begin;
+  for (auto it = begin + 1; it != end; ++it)
+  {
+    delta = coding::EncodePointDeltaAsUint(*it, *prevIt);
+    WriteVarUint(w, delta);
+    prevIt = it;
+  }
 }
 
 void CentersTableBuilder::Freeze(Writer & writer) const
 {
-  // Each center is encoded as delta from some prediction.
-  // For the first center in the block map base point is used as a prediction, for all other
-  // centers in the block previous center is used as a prediction.
-  auto const writeBlockCallback = [&](Writer & w, vector<m2::PointU>::const_iterator begin,
-                                      vector<m2::PointU>::const_iterator end) {
-    uint64_t delta = coding::EncodePointDeltaAsUint(*begin, m_codingParams.GetBasePoint());
-    WriteVarUint(w, delta);
-    auto prevIt = begin;
-    for (auto it = begin + 1; it != end; ++it)
-    {
-      delta = coding::EncodePointDeltaAsUint(*it, *prevIt);
-      WriteVarUint(w, delta);
-      prevIt = it;
-    }
-  };
+  size_t startOffset = writer.Pos();
+  CHECK(coding::IsAlign8(startOffset), ());
 
-  m_builder.Freeze(writer, writeBlockCallback);
+  CentersTable::Header header;
+  header.Serialize(writer);
+
+  uint64_t bytesWritten = writer.Pos();
+  coding::WritePadding(writer, bytesWritten);
+
+  header.m_geometryParamsOffset = base::asserted_cast<uint32_t>(writer.Pos() - startOffset);
+  m_codingParams.Save(writer);
+  auto leftBottom = PointDToPointU(m_limitRect.LeftBottom(), kPointCoordBits);
+  WriteToSink(writer, leftBottom.x);
+  WriteToSink(writer, leftBottom.y);
+  auto rightTop = PointDToPointU(m_limitRect.RightTop(), kPointCoordBits);
+  WriteToSink(writer, rightTop.x);
+  WriteToSink(writer, rightTop.y);
+  header.m_geometryParamsSize =
+      base::asserted_cast<uint32_t>(writer.Pos() - header.m_geometryParamsOffset - startOffset);
+
+  bytesWritten = writer.Pos();
+  coding::WritePadding(writer, bytesWritten);
+
+  header.m_centersOffset = base::asserted_cast<uint32_t>(writer.Pos() - startOffset);
+  m_builder.Freeze(writer, [&](auto & w, auto begin, auto end) { WriteBlock(w, begin, end); });
+  header.m_centersSize =
+      base::asserted_cast<uint32_t>(writer.Pos() - header.m_centersOffset - startOffset);
+
+  auto const endOffset = writer.Pos();
+  writer.Seek(startOffset);
+  header.Serialize(writer);
+  writer.Seek(endOffset);
+}
+
+void CentersTableBuilder::SetGeometryCodingParamsV0ForTests(
+    serial::GeometryCodingParams const & codingParams)
+{
+  m_codingParams = codingParams;
+}
+
+void CentersTableBuilder::PutV0ForTests(uint32_t featureId, m2::PointD const & center)
+{
+  m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits()));
+}
+
+void CentersTableBuilder::FreezeV0ForTests(Writer & writer) const
+{
+  m_builder.Freeze(writer, [&](auto & w, auto begin, auto end) { WriteBlock(w, begin, end); });
 }
 }  // namespace search

indexer/centers_table.hpp

@@ -2,6 +2,7 @@
 
 #include "coding/geometry_coding.hpp"
 #include "coding/map_uint32_to_val.hpp"
+#include "coding/point_coding.hpp"
 
 #include "geometry/point2d.hpp"
 
@@ -19,6 +20,36 @@ namespace search
 class CentersTable
 {
 public:
+  enum class Version : uint8_t
+  {
+    V0 = 0,
+    V1 = 1,
+    Latest = V1
+  };
+
+  struct Header
+  {
+    template <typename Sink>
+    void Serialize(Sink & sink) const
+    {
+      CHECK_EQUAL(static_cast<uint8_t>(m_version), static_cast<uint8_t>(Version::V1), ());
+      WriteToSink(sink, static_cast<uint8_t>(m_version));
+      WriteToSink(sink, m_geometryParamsOffset);
+      WriteToSink(sink, m_geometryParamsSize);
+      WriteToSink(sink, m_centersOffset);
+      WriteToSink(sink, m_centersSize);
+    }
+
+    void Read(Reader & reader);
+
+    Version m_version = Version::Latest;
+    // All offsets are relative to the start of the section (offset of header is zero).
+    uint32_t m_geometryParamsOffset = 0;
+    uint32_t m_geometryParamsSize = 0;
+    uint32_t m_centersOffset = 0;
+    uint32_t m_centersSize = 0;
+  };
+
   // Tries to get |center| of the feature identified by |id|.  Returns
   // false if table does not have entry for the feature.
   WARN_UNUSED_RESULT bool Get(uint32_t id, m2::PointD & center);
@@ -26,30 +57,40 @@ public:
   // Loads CentersTable instance. Note that |reader| must be alive
   // until the destruction of loaded table. Returns nullptr if
   // CentersTable can't be loaded.
-  static std::unique_ptr<CentersTable> Load(Reader & reader,
-                                            serial::GeometryCodingParams const & codingParams);
+  static std::unique_ptr<CentersTable> LoadV0(Reader & reader,
+                                              serial::GeometryCodingParams const & codingParams);
+
+  static std::unique_ptr<CentersTable> LoadV1(Reader & reader);
+
 private:
   using Map = MapUint32ToValue<m2::PointU>;
 
-  bool Init(Reader & reader, serial::GeometryCodingParams const & codingParams);
+  bool Init(Reader & reader, serial::GeometryCodingParams const & codingParams,
+            m2::RectD const & limitRect);
 
   serial::GeometryCodingParams m_codingParams;
   std::unique_ptr<Map> m_map;
+  std::unique_ptr<Reader> m_centersSubreader;
+  m2::RectD m_limitRect;
+  Version m_version = Version::Latest;
 };
 
 class CentersTableBuilder
 {
 public:
-  inline void SetGeometryCodingParams(serial::GeometryCodingParams const & codingParams)
-  {
-    m_codingParams = codingParams;
-  }
-
+  void SetGeometryParams(m2::RectD const & limitRect, double pointAccuracy = kMwmPointAccuracy);
   void Put(uint32_t featureId, m2::PointD const & center);
+  void WriteBlock(Writer & w, std::vector<m2::PointU>::const_iterator begin,
+                  std::vector<m2::PointU>::const_iterator end) const;
   void Freeze(Writer & writer) const;
 
+  void SetGeometryCodingParamsV0ForTests(serial::GeometryCodingParams const & codingParams);
+  void PutV0ForTests(uint32_t featureId, m2::PointD const & center);
+  void FreezeV0ForTests(Writer & writer) const;
+
 private:
   serial::GeometryCodingParams m_codingParams;
+  m2::RectD m_limitRect;
   MapUint32ToValueBuilder<m2::PointU> m_builder;
 };
 }  // namespace search

indexer/indexer_tests/centers_table_test.cpp

@@ -37,8 +37,6 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke)
 {
   string const kMap = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwm");
 
-  feature::DataHeader header(kMap);
-  auto const codingParams = header.GetDefGeometryCodingParams();
 
   FeaturesVectorTest fv(kMap);
 
@@ -46,8 +44,9 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke)
 
   {
     CentersTableBuilder builder;
+    feature::DataHeader header(kMap);
 
-    builder.SetGeometryCodingParams(codingParams);
+    builder.SetGeometryParams(header.GetBounds());
     fv.GetVector().ForEach(
         [&](FeatureType & ft, uint32_t id) { builder.Put(id, feature::GetCenter(ft)); });
 
@@ -57,7 +56,45 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke)
 
   {
     MemReader reader(buffer.data(), buffer.size());
-    auto table = CentersTable::Load(reader, codingParams);
+    auto table = CentersTable::LoadV1(reader);
+    TEST(table.get(), ());
+
+    fv.GetVector().ForEach([&](FeatureType & ft, uint32_t id) {
+      m2::PointD actual;
+      TEST(table->Get(id, actual), ());
+
+      m2::PointD expected = feature::GetCenter(ft);
+
+      TEST_LESS_OR_EQUAL(MercatorBounds::DistanceOnEarth(actual, expected), 1, (id));
+    });
+  }
+}
+
+UNIT_CLASS_TEST(CentersTableTest, SmokeV0)
+{
+  string const kMap = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwm");
+
+  FeaturesVectorTest fv(kMap);
+
+  feature::DataHeader header(kMap);
+  auto const codingParams = header.GetDefGeometryCodingParams();
+
+  TBuffer buffer;
+
+  {
+    CentersTableBuilder builder;
+
+    builder.SetGeometryCodingParamsV0ForTests(codingParams);
+    fv.GetVector().ForEach(
+        [&](FeatureType & ft, uint32_t id) { builder.PutV0ForTests(id, feature::GetCenter(ft)); });
+
+    MemWriter<TBuffer> writer(buffer);
+    builder.FreezeV0ForTests(writer);
+  }
+
+  {
+    MemReader reader(buffer.data(), buffer.size());
+    auto table = CentersTable::LoadV0(reader, codingParams);
     TEST(table.get(), ());
 
     fv.GetVector().ForEach([&](FeatureType & ft, uint32_t id) {
@@ -74,15 +111,13 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke)
 UNIT_CLASS_TEST(CentersTableTest, Subset)
 {
   vector<pair<uint32_t, m2::PointD>> const features = {
-      {1, m2::PointD(0, 0)}, {5, m2::PointD(1, 1)}, {10, m2::PointD(2, 2)}};
-
-  serial::GeometryCodingParams codingParams;
+      {1, m2::PointD(0.0, 0.0)}, {5, m2::PointD(1.0, 1.0)}, {10, m2::PointD(2.0, 2.0)}};
 
   TBuffer buffer;
   {
     CentersTableBuilder builder;
 
-    builder.SetGeometryCodingParams(codingParams);
+    builder.SetGeometryParams({{0.0, 0.0}, {2.0, 2.0}});
     for (auto const & feature : features)
       builder.Put(feature.first, feature.second);
 
@@ -92,7 +127,7 @@ UNIT_CLASS_TEST(CentersTableTest, Subset)
 
   {
     MemReader reader(buffer.data(), buffer.size());
-    auto table = CentersTable::Load(reader, codingParams);
+    auto table = CentersTable::LoadV1(reader);
     TEST(table.get(), ());
 
     uint32_t i = 0;

platform/mwm_traits.cpp

@@ -26,6 +26,18 @@ MwmTraits::HouseToStreetTableFormat MwmTraits::GetHouseToStreetTableFormat() con
   return HouseToStreetTableFormat::EliasFanoMap;
 }
 
+MwmTraits::CentersTableFormat MwmTraits::GetCentersTableFormat() const
+{
+  if (GetFormat() < version::Format::v9)
+    return CentersTableFormat::PlainEliasFanoMap;
+
+  uint32_t constexpr kLastVersionWithPlainEliasFanoMap = 190923;
+  if (GetVersion() <= kLastVersionWithPlainEliasFanoMap)
+    return CentersTableFormat::PlainEliasFanoMap;
+
+  return CentersTableFormat::EliasFanoMapWithHeader;
+}
+
 bool MwmTraits::HasOffsetsTable() const { return GetFormat() >= version::Format::v6; }
 
 bool MwmTraits::HasCrossMwmSection() const { return GetFormat() >= version::Format::v9; }

platform/mwm_traits.hpp

@@ -41,12 +41,23 @@ public:
     Unknown
   };
 
+  enum class CentersTableFormat
+  {
+    // Centers table encoded without any header. Coding params from mwm header used.
+    PlainEliasFanoMap,
+
+    // Centers table has it's own header with version and coding params.
+    EliasFanoMapWithHeader,
+  };
+
   explicit MwmTraits(MwmVersion const & version);
 
   SearchIndexFormat GetSearchIndexFormat() const;
 
   HouseToStreetTableFormat GetHouseToStreetTableFormat() const;
 
+  CentersTableFormat GetCentersTableFormat() const;
+
   bool HasOffsetsTable() const;
 
   bool HasCrossMwmSection() const;

search/lazy_centers_table.cpp

@@ -2,6 +2,8 @@
 
 #include "indexer/mwm_set.hpp"
 
+#include "platform/mwm_traits.hpp"
+
 #include "defines.hpp"
 
 namespace search
@@ -29,8 +31,23 @@ void LazyCentersTable::EnsureTableLoaded()
     return;
   }
 
-  m_table =
-      CentersTable::Load(*m_reader.GetPtr(), m_value.GetHeader().GetDefGeometryCodingParams());
+  version::MwmTraits traits(m_value.GetMwmVersion());
+  auto const format = traits.GetCentersTableFormat();
+
+  if (format == version::MwmTraits::CentersTableFormat::PlainEliasFanoMap)
+  {
+    m_table =
+        CentersTable::LoadV0(*m_reader.GetPtr(), m_value.GetHeader().GetDefGeometryCodingParams());
+  }
+  else if (format == version::MwmTraits::CentersTableFormat::EliasFanoMapWithHeader)
+  {
+    m_table = CentersTable::LoadV1(*m_reader.GetPtr());
+  }
+  else
+  {
+    CHECK(false, ("Unknown centers table format."));
+  }
+
   if (m_table)
     m_state = STATE_LOADED;
   else

search/postcode_points.cpp

@@ -2,6 +2,8 @@
 
 #include "indexer/trie_reader.hpp"
 
+#include "platform/mwm_traits.hpp"
+
 #include "coding/reader_wrapper.hpp"
 
 #include "geometry/mercator.hpp"
@@ -40,10 +42,14 @@ PostcodePoints::PostcodePoints(MwmValue const & value)
       SubReaderWrapper<Reader>(m_trieSubReader.get()), SingleValueSerializer<Uint64IndexValue>());
   CHECK(m_root, ());
 
+  version::MwmTraits traits(value.GetMwmVersion());
+  auto const format = traits.GetCentersTableFormat();
+  CHECK(format == version::MwmTraits::CentersTableFormat::EliasFanoMapWithHeader,
+        ("Unexpected format."));
+
   m_pointsSubReader =
       reader.GetPtr()->CreateSubReader(m_header.m_pointsOffset, m_header.m_pointsSize);
-  auto const codingParams = value.GetHeader().GetDefGeometryCodingParams();
-  m_points = CentersTable::Load(*m_pointsSubReader, codingParams);
+  m_points = CentersTable::LoadV1(*m_pointsSubReader);
   CHECK(m_points, ());
 }
 

search/search_integration_tests/pre_ranker_test.cpp

@@ -168,7 +168,7 @@ UNIT_CLASS_TEST(PreRankerTest, Smoke)
     TEST_LESS(index, pois.size(), ());
 
     TEST(!checked[index], (index));
-    TEST(base::AlmostEqualAbs(distances[index], results[i].GetDistance(), 1e-3),
+    TEST(base::AlmostEqualAbs(distances[index], results[i].GetDistance(), 1.0),
          (distances[index], results[i].GetDistance()));
     checked[index] = true;
   }