From 549cd65ad592bda966db509db894567aa530a9a4 Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Thu, 26 Sep 2019 12:00:24 +0300 Subject: [PATCH] [indexer] Optimize centers table to use less CoordBits. --- generator/centers_table_builder.cpp | 2 +- generator/search_index_builder.cpp | 3 +- indexer/centers_table.cpp | 149 +++++++++++++++--- indexer/centers_table.hpp | 57 ++++++- indexer/indexer_tests/centers_table_test.cpp | 53 +++++-- platform/mwm_traits.cpp | 12 ++ platform/mwm_traits.hpp | 11 ++ search/lazy_centers_table.cpp | 21 ++- search/postcode_points.cpp | 10 +- .../pre_ranker_test.cpp | 2 +- 10 files changed, 274 insertions(+), 46 deletions(-) diff --git a/generator/centers_table_builder.cpp b/generator/centers_table_builder.cpp index 08cca52322..0937721d47 100644 --- a/generator/centers_table_builder.cpp +++ b/generator/centers_table_builder.cpp @@ -52,7 +52,7 @@ bool BuildCentersTableFromDataFile(std::string const & filename, bool forceRebui feature::DataHeader const header(rcont); FeaturesVector const features(rcont, header, table.get()); - builder.SetGeometryCodingParams(header.GetDefGeometryCodingParams()); + builder.SetGeometryParams(header.GetBounds()); features.ForEach([&](FeatureType & ft, uint32_t featureId) { builder.Put(featureId, feature::GetCenter(ft)); }); diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index c6758c3611..88647dc1d6 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -727,7 +727,8 @@ bool BuildPostcodesImpl(FilesContainerR & container, storage::CountryId const & { search::CentersTableBuilder builder; - builder.SetGeometryCodingParams(feature::DataHeader(container).GetDefGeometryCodingParams()); + + builder.SetGeometryParams(feature::DataHeader(container).GetBounds()); for (size_t i = 0; i < valueMapping.size(); ++i) builder.Put(base::asserted_cast(i), valueMapping[i]); diff --git a/indexer/centers_table.cpp b/indexer/centers_table.cpp index a27361f2ae..99bc253dd0 100644 --- a/indexer/centers_table.cpp +++ b/indexer/centers_table.cpp @@ -26,30 +26,80 @@ using namespace std; namespace search { +void CentersTable::Header::Read(Reader & reader) +{ + NonOwningReaderSource source(reader); + m_version = static_cast(ReadPrimitiveFromSource(source)); + CHECK_EQUAL(static_cast(m_version), static_cast(Version::V1), ()); + m_geometryParamsOffset = ReadPrimitiveFromSource(source); + m_geometryParamsSize = ReadPrimitiveFromSource(source); + m_centersOffset = ReadPrimitiveFromSource(source); + m_centersSize = ReadPrimitiveFromSource(source); +} + bool CentersTable::Get(uint32_t id, m2::PointD & center) { m2::PointU pointu; if (!m_map->Get(id, pointu)) return false; - center = PointUToPointD(pointu, m_codingParams.GetCoordBits()); + if (m_version == Version::V0) + center = PointUToPointD(pointu, m_codingParams.GetCoordBits()); + else if (m_version == Version::V1) + center = PointUToPointD(pointu, m_codingParams.GetCoordBits(), m_limitRect); + else + CHECK(false, ("Unknown CentersTable format.")); + return true; } // CentersTable ------------------------------------------------------------------------------------ // static -unique_ptr CentersTable::Load(Reader & reader, - serial::GeometryCodingParams const & codingParams) +unique_ptr CentersTable::LoadV0(Reader & reader, + serial::GeometryCodingParams const & codingParams) { auto table = make_unique(); - if (!table->Init(reader, codingParams)) + table->m_version = Version::V0; + if (!table->Init(reader, codingParams, {} /* limitRect */)) return {}; return table; } -bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & codingParams) +unique_ptr CentersTable::LoadV1(Reader & reader) +{ + auto table = make_unique(); + table->m_version = Version::V1; + + Header header; + header.Read(reader); + + auto geometryParamsSubreader = + reader.CreateSubReader(header.m_geometryParamsOffset, header.m_geometryParamsSize); + if (!geometryParamsSubreader) + return {}; + NonOwningReaderSource geometryParamsSource(*geometryParamsSubreader); + serial::GeometryCodingParams codingParams; + codingParams.Load(geometryParamsSource); + auto minX = ReadPrimitiveFromSource(geometryParamsSource); + auto minY = ReadPrimitiveFromSource(geometryParamsSource); + auto maxX = ReadPrimitiveFromSource(geometryParamsSource); + auto maxY = ReadPrimitiveFromSource(geometryParamsSource); + m2::RectD limitRect(PointUToPointD({minX, minY}, kPointCoordBits), + PointUToPointD({maxX, maxY}, kPointCoordBits)); + + table->m_centersSubreader = reader.CreateSubReader(header.m_centersOffset, header.m_centersSize); + if (!table->m_centersSubreader) + return {}; + if (!table->Init(*(table->m_centersSubreader), codingParams, limitRect)) + return {}; + return table; +} + +bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & codingParams, + m2::RectD const & limitRect) { m_codingParams = codingParams; + m_limitRect = limitRect; // Decodes block encoded by writeBlockCallback from CentersTableBuilder::Freeze. auto const readBlockCallback = [&](NonOwningReaderSource & source, uint32_t blockSize, vector & values) { @@ -69,29 +119,84 @@ bool CentersTable::Init(Reader & reader, serial::GeometryCodingParams const & co } // CentersTableBuilder ----------------------------------------------------------------------------- +void CentersTableBuilder::SetGeometryParams(m2::RectD const & limitRect, double pointAccuracy) +{ + auto const coordBits = GetCoordBits(limitRect, pointAccuracy); + m_codingParams = serial::GeometryCodingParams(coordBits, limitRect.Center()); + m_limitRect = limitRect; +} + void CentersTableBuilder::Put(uint32_t featureId, m2::PointD const & center) { - m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits())); + m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits(), m_limitRect)); +} + +// Each center is encoded as delta from some prediction. +// For the first center in the block map base point is used as a prediction, for all other +// centers in the block previous center is used as a prediction. +void CentersTableBuilder::WriteBlock(Writer & w, vector::const_iterator begin, + vector::const_iterator end) const +{ + uint64_t delta = coding::EncodePointDeltaAsUint(*begin, m_codingParams.GetBasePoint()); + WriteVarUint(w, delta); + auto prevIt = begin; + for (auto it = begin + 1; it != end; ++it) + { + delta = coding::EncodePointDeltaAsUint(*it, *prevIt); + WriteVarUint(w, delta); + prevIt = it; + } } void CentersTableBuilder::Freeze(Writer & writer) const { - // Each center is encoded as delta from some prediction. - // For the first center in the block map base point is used as a prediction, for all other - // centers in the block previous center is used as a prediction. - auto const writeBlockCallback = [&](Writer & w, vector::const_iterator begin, - vector::const_iterator end) { - uint64_t delta = coding::EncodePointDeltaAsUint(*begin, m_codingParams.GetBasePoint()); - WriteVarUint(w, delta); - auto prevIt = begin; - for (auto it = begin + 1; it != end; ++it) - { - delta = coding::EncodePointDeltaAsUint(*it, *prevIt); - WriteVarUint(w, delta); - prevIt = it; - } - }; + size_t startOffset = writer.Pos(); + CHECK(coding::IsAlign8(startOffset), ()); - m_builder.Freeze(writer, writeBlockCallback); + CentersTable::Header header; + header.Serialize(writer); + + uint64_t bytesWritten = writer.Pos(); + coding::WritePadding(writer, bytesWritten); + + header.m_geometryParamsOffset = base::asserted_cast(writer.Pos() - startOffset); + m_codingParams.Save(writer); + auto leftBottom = PointDToPointU(m_limitRect.LeftBottom(), kPointCoordBits); + WriteToSink(writer, leftBottom.x); + WriteToSink(writer, leftBottom.y); + auto rightTop = PointDToPointU(m_limitRect.RightTop(), kPointCoordBits); + WriteToSink(writer, rightTop.x); + WriteToSink(writer, rightTop.y); + header.m_geometryParamsSize = + base::asserted_cast(writer.Pos() - header.m_geometryParamsOffset - startOffset); + + bytesWritten = writer.Pos(); + coding::WritePadding(writer, bytesWritten); + + header.m_centersOffset = base::asserted_cast(writer.Pos() - startOffset); + m_builder.Freeze(writer, [&](auto & w, auto begin, auto end) { WriteBlock(w, begin, end); }); + header.m_centersSize = + base::asserted_cast(writer.Pos() - header.m_centersOffset - startOffset); + + auto const endOffset = writer.Pos(); + writer.Seek(startOffset); + header.Serialize(writer); + writer.Seek(endOffset); +} + +void CentersTableBuilder::SetGeometryCodingParamsV0ForTests( + serial::GeometryCodingParams const & codingParams) +{ + m_codingParams = codingParams; +} + +void CentersTableBuilder::PutV0ForTests(uint32_t featureId, m2::PointD const & center) +{ + m_builder.Put(featureId, PointDToPointU(center, m_codingParams.GetCoordBits())); +} + +void CentersTableBuilder::FreezeV0ForTests(Writer & writer) const +{ + m_builder.Freeze(writer, [&](auto & w, auto begin, auto end) { WriteBlock(w, begin, end); }); } } // namespace search diff --git a/indexer/centers_table.hpp b/indexer/centers_table.hpp index 4df749b088..01dcc1dd0e 100644 --- a/indexer/centers_table.hpp +++ b/indexer/centers_table.hpp @@ -2,6 +2,7 @@ #include "coding/geometry_coding.hpp" #include "coding/map_uint32_to_val.hpp" +#include "coding/point_coding.hpp" #include "geometry/point2d.hpp" @@ -19,6 +20,36 @@ namespace search class CentersTable { public: + enum class Version : uint8_t + { + V0 = 0, + V1 = 1, + Latest = V1 + }; + + struct Header + { + template + void Serialize(Sink & sink) const + { + CHECK_EQUAL(static_cast(m_version), static_cast(Version::V1), ()); + WriteToSink(sink, static_cast(m_version)); + WriteToSink(sink, m_geometryParamsOffset); + WriteToSink(sink, m_geometryParamsSize); + WriteToSink(sink, m_centersOffset); + WriteToSink(sink, m_centersSize); + } + + void Read(Reader & reader); + + Version m_version = Version::Latest; + // All offsets are relative to the start of the section (offset of header is zero). + uint32_t m_geometryParamsOffset = 0; + uint32_t m_geometryParamsSize = 0; + uint32_t m_centersOffset = 0; + uint32_t m_centersSize = 0; + }; + // Tries to get |center| of the feature identified by |id|. Returns // false if table does not have entry for the feature. WARN_UNUSED_RESULT bool Get(uint32_t id, m2::PointD & center); @@ -26,30 +57,40 @@ public: // Loads CentersTable instance. Note that |reader| must be alive // until the destruction of loaded table. Returns nullptr if // CentersTable can't be loaded. - static std::unique_ptr Load(Reader & reader, - serial::GeometryCodingParams const & codingParams); + static std::unique_ptr LoadV0(Reader & reader, + serial::GeometryCodingParams const & codingParams); + + static std::unique_ptr LoadV1(Reader & reader); + private: using Map = MapUint32ToValue; - bool Init(Reader & reader, serial::GeometryCodingParams const & codingParams); + bool Init(Reader & reader, serial::GeometryCodingParams const & codingParams, + m2::RectD const & limitRect); serial::GeometryCodingParams m_codingParams; std::unique_ptr m_map; + std::unique_ptr m_centersSubreader; + m2::RectD m_limitRect; + Version m_version = Version::Latest; }; class CentersTableBuilder { public: - inline void SetGeometryCodingParams(serial::GeometryCodingParams const & codingParams) - { - m_codingParams = codingParams; - } - + void SetGeometryParams(m2::RectD const & limitRect, double pointAccuracy = kMwmPointAccuracy); void Put(uint32_t featureId, m2::PointD const & center); + void WriteBlock(Writer & w, std::vector::const_iterator begin, + std::vector::const_iterator end) const; void Freeze(Writer & writer) const; + void SetGeometryCodingParamsV0ForTests(serial::GeometryCodingParams const & codingParams); + void PutV0ForTests(uint32_t featureId, m2::PointD const & center); + void FreezeV0ForTests(Writer & writer) const; + private: serial::GeometryCodingParams m_codingParams; + m2::RectD m_limitRect; MapUint32ToValueBuilder m_builder; }; } // namespace search diff --git a/indexer/indexer_tests/centers_table_test.cpp b/indexer/indexer_tests/centers_table_test.cpp index 5c9e01f249..1af9d10edf 100644 --- a/indexer/indexer_tests/centers_table_test.cpp +++ b/indexer/indexer_tests/centers_table_test.cpp @@ -37,8 +37,6 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke) { string const kMap = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwm"); - feature::DataHeader header(kMap); - auto const codingParams = header.GetDefGeometryCodingParams(); FeaturesVectorTest fv(kMap); @@ -46,8 +44,9 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke) { CentersTableBuilder builder; + feature::DataHeader header(kMap); - builder.SetGeometryCodingParams(codingParams); + builder.SetGeometryParams(header.GetBounds()); fv.GetVector().ForEach( [&](FeatureType & ft, uint32_t id) { builder.Put(id, feature::GetCenter(ft)); }); @@ -57,7 +56,45 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke) { MemReader reader(buffer.data(), buffer.size()); - auto table = CentersTable::Load(reader, codingParams); + auto table = CentersTable::LoadV1(reader); + TEST(table.get(), ()); + + fv.GetVector().ForEach([&](FeatureType & ft, uint32_t id) { + m2::PointD actual; + TEST(table->Get(id, actual), ()); + + m2::PointD expected = feature::GetCenter(ft); + + TEST_LESS_OR_EQUAL(MercatorBounds::DistanceOnEarth(actual, expected), 1, (id)); + }); + } +} + +UNIT_CLASS_TEST(CentersTableTest, SmokeV0) +{ + string const kMap = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwm"); + + FeaturesVectorTest fv(kMap); + + feature::DataHeader header(kMap); + auto const codingParams = header.GetDefGeometryCodingParams(); + + TBuffer buffer; + + { + CentersTableBuilder builder; + + builder.SetGeometryCodingParamsV0ForTests(codingParams); + fv.GetVector().ForEach( + [&](FeatureType & ft, uint32_t id) { builder.PutV0ForTests(id, feature::GetCenter(ft)); }); + + MemWriter writer(buffer); + builder.FreezeV0ForTests(writer); + } + + { + MemReader reader(buffer.data(), buffer.size()); + auto table = CentersTable::LoadV0(reader, codingParams); TEST(table.get(), ()); fv.GetVector().ForEach([&](FeatureType & ft, uint32_t id) { @@ -74,15 +111,13 @@ UNIT_CLASS_TEST(CentersTableTest, Smoke) UNIT_CLASS_TEST(CentersTableTest, Subset) { vector> const features = { - {1, m2::PointD(0, 0)}, {5, m2::PointD(1, 1)}, {10, m2::PointD(2, 2)}}; - - serial::GeometryCodingParams codingParams; + {1, m2::PointD(0.0, 0.0)}, {5, m2::PointD(1.0, 1.0)}, {10, m2::PointD(2.0, 2.0)}}; TBuffer buffer; { CentersTableBuilder builder; - builder.SetGeometryCodingParams(codingParams); + builder.SetGeometryParams({{0.0, 0.0}, {2.0, 2.0}}); for (auto const & feature : features) builder.Put(feature.first, feature.second); @@ -92,7 +127,7 @@ UNIT_CLASS_TEST(CentersTableTest, Subset) { MemReader reader(buffer.data(), buffer.size()); - auto table = CentersTable::Load(reader, codingParams); + auto table = CentersTable::LoadV1(reader); TEST(table.get(), ()); uint32_t i = 0; diff --git a/platform/mwm_traits.cpp b/platform/mwm_traits.cpp index b51fbdc7f4..4cc5ac3e23 100644 --- a/platform/mwm_traits.cpp +++ b/platform/mwm_traits.cpp @@ -26,6 +26,18 @@ MwmTraits::HouseToStreetTableFormat MwmTraits::GetHouseToStreetTableFormat() con return HouseToStreetTableFormat::EliasFanoMap; } +MwmTraits::CentersTableFormat MwmTraits::GetCentersTableFormat() const +{ + if (GetFormat() < version::Format::v9) + return CentersTableFormat::PlainEliasFanoMap; + + uint32_t constexpr kLastVersionWithPlainEliasFanoMap = 190923; + if (GetVersion() <= kLastVersionWithPlainEliasFanoMap) + return CentersTableFormat::PlainEliasFanoMap; + + return CentersTableFormat::EliasFanoMapWithHeader; +} + bool MwmTraits::HasOffsetsTable() const { return GetFormat() >= version::Format::v6; } bool MwmTraits::HasCrossMwmSection() const { return GetFormat() >= version::Format::v9; } diff --git a/platform/mwm_traits.hpp b/platform/mwm_traits.hpp index 9bf444a099..1f3027ee56 100644 --- a/platform/mwm_traits.hpp +++ b/platform/mwm_traits.hpp @@ -41,12 +41,23 @@ public: Unknown }; + enum class CentersTableFormat + { + // Centers table encoded without any header. Coding params from mwm header used. + PlainEliasFanoMap, + + // Centers table has it's own header with version and coding params. + EliasFanoMapWithHeader, + }; + explicit MwmTraits(MwmVersion const & version); SearchIndexFormat GetSearchIndexFormat() const; HouseToStreetTableFormat GetHouseToStreetTableFormat() const; + CentersTableFormat GetCentersTableFormat() const; + bool HasOffsetsTable() const; bool HasCrossMwmSection() const; diff --git a/search/lazy_centers_table.cpp b/search/lazy_centers_table.cpp index 99a06d9179..ae7e65875b 100644 --- a/search/lazy_centers_table.cpp +++ b/search/lazy_centers_table.cpp @@ -2,6 +2,8 @@ #include "indexer/mwm_set.hpp" +#include "platform/mwm_traits.hpp" + #include "defines.hpp" namespace search @@ -29,8 +31,23 @@ void LazyCentersTable::EnsureTableLoaded() return; } - m_table = - CentersTable::Load(*m_reader.GetPtr(), m_value.GetHeader().GetDefGeometryCodingParams()); + version::MwmTraits traits(m_value.GetMwmVersion()); + auto const format = traits.GetCentersTableFormat(); + + if (format == version::MwmTraits::CentersTableFormat::PlainEliasFanoMap) + { + m_table = + CentersTable::LoadV0(*m_reader.GetPtr(), m_value.GetHeader().GetDefGeometryCodingParams()); + } + else if (format == version::MwmTraits::CentersTableFormat::EliasFanoMapWithHeader) + { + m_table = CentersTable::LoadV1(*m_reader.GetPtr()); + } + else + { + CHECK(false, ("Unknown centers table format.")); + } + if (m_table) m_state = STATE_LOADED; else diff --git a/search/postcode_points.cpp b/search/postcode_points.cpp index 9ae470370b..7df573ed76 100644 --- a/search/postcode_points.cpp +++ b/search/postcode_points.cpp @@ -2,6 +2,8 @@ #include "indexer/trie_reader.hpp" +#include "platform/mwm_traits.hpp" + #include "coding/reader_wrapper.hpp" #include "geometry/mercator.hpp" @@ -40,10 +42,14 @@ PostcodePoints::PostcodePoints(MwmValue const & value) SubReaderWrapper(m_trieSubReader.get()), SingleValueSerializer()); CHECK(m_root, ()); + version::MwmTraits traits(value.GetMwmVersion()); + auto const format = traits.GetCentersTableFormat(); + CHECK(format == version::MwmTraits::CentersTableFormat::EliasFanoMapWithHeader, + ("Unexpected format.")); + m_pointsSubReader = reader.GetPtr()->CreateSubReader(m_header.m_pointsOffset, m_header.m_pointsSize); - auto const codingParams = value.GetHeader().GetDefGeometryCodingParams(); - m_points = CentersTable::Load(*m_pointsSubReader, codingParams); + m_points = CentersTable::LoadV1(*m_pointsSubReader); CHECK(m_points, ()); } diff --git a/search/search_integration_tests/pre_ranker_test.cpp b/search/search_integration_tests/pre_ranker_test.cpp index 69be669a6d..f1a53da730 100644 --- a/search/search_integration_tests/pre_ranker_test.cpp +++ b/search/search_integration_tests/pre_ranker_test.cpp @@ -168,7 +168,7 @@ UNIT_CLASS_TEST(PreRankerTest, Smoke) TEST_LESS(index, pois.size(), ()); TEST(!checked[index], (index)); - TEST(base::AlmostEqualAbs(distances[index], results[i].GetDistance(), 1e-3), + TEST(base::AlmostEqualAbs(distances[index], results[i].GetDistance(), 1.0), (distances[index], results[i].GetDistance())); checked[index] = true; }