diff --git a/generator/centers_table_builder.cpp b/generator/centers_table_builder.cpp index a1097f3703..b58608b1aa 100644 --- a/generator/centers_table_builder.cpp +++ b/generator/centers_table_builder.cpp @@ -10,7 +10,8 @@ #include "coding/file_container.hpp" #include "platform/mwm_traits.hpp" -#include "platform/platform.hpp" + +#include "base/exception.hpp" #include "defines.hpp" @@ -18,41 +19,49 @@ namespace indexer { bool BuildCentersTableFromDataFile(string const & filename, bool forceRebuild) { - search::CentersTableBuilder builder; - + try { - auto const & platform = GetPlatform(); - FilesContainerR rcont(platform.GetReader(filename, "f")); - if (!forceRebuild && rcont.IsExist(CENTERS_FILE_TAG)) - return true; + search::CentersTableBuilder builder; - feature::DataHeader header(rcont); + { + FilesContainerR rcont(filename); + if (!forceRebuild && rcont.IsExist(CENTERS_FILE_TAG)) + return true; - version::MwmTraits traits(header.GetFormat()); - if (!traits.HasOffsetsTable()) - { - LOG(LERROR, (filename, "does not have an offsets table!")); - return false; - } - auto table = feature::FeaturesOffsetsTable::Load(rcont); - if (!table) - { - LOG(LERROR, ("Can't load offsets table from:", filename)); - return false; + feature::DataHeader header(rcont); + + version::MwmTraits const traits(header.GetFormat()); + if (!traits.HasOffsetsTable()) + { + LOG(LERROR, (filename, "does not have an offsets table!")); + return false; + } + + auto const table = feature::FeaturesOffsetsTable::Load(rcont); + if (!table) + { + LOG(LERROR, ("Can't load offsets table from:", filename)); + return false; + } + + FeaturesVector const features(rcont, header, table.get()); + + builder.SetCodingParams(header.GetDefCodingParams()); + features.ForEach([&](FeatureType & ft, uint32_t featureId) { + builder.Put(featureId, feature::GetCenter(ft)); + }); } - FeaturesVector features(rcont, header, table.get()); - - builder.SetCodingParams(header.GetDefCodingParams()); - features.ForEach([&](FeatureType & ft, uint32_t featureId) { - builder.Put(featureId, feature::GetCenter(ft)); - }); + { + FilesContainerW writeContainer(filename, FileWriter::OP_WRITE_EXISTING); + FileWriter writer = writeContainer.GetWriter(CENTERS_FILE_TAG); + builder.Freeze(writer); + } } - + catch (RootException const & e) { - FilesContainerW writeContainer(filename, FileWriter::OP_WRITE_EXISTING); - FileWriter writer = writeContainer.GetWriter(CENTERS_FILE_TAG); - builder.Freeze(writer); + LOG(LERROR, ("Failed to build centers table:", e.Msg())); + return false; } return true; diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 9adb00672d..c7af0f0a85 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -137,8 +137,8 @@ int main(int argc, char ** argv) // Load classificator only when necessary. if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry || - FLAGS_generate_index || FLAGS_generate_search_index || FLAGS_generate_centers_table || - FLAGS_calc_statistics || FLAGS_type_statistics || FLAGS_dump_types || FLAGS_dump_prefixes || + FLAGS_generate_index || FLAGS_generate_search_index || FLAGS_calc_statistics || + FLAGS_type_statistics || FLAGS_dump_types || FLAGS_dump_prefixes || FLAGS_dump_feature_names != "" || FLAGS_check_mwm) { classificator::Load(); diff --git a/indexer/centers_table.cpp b/indexer/centers_table.cpp index 72101e618c..f0a1fa7d6a 100644 --- a/indexer/centers_table.cpp +++ b/indexer/centers_table.cpp @@ -272,7 +272,7 @@ void CentersTableBuilder::Put(uint32_t featureId, m2::PointD const & center) m_ids.push_back(featureId); } -void CentersTableBuilder::Freeze(Writer & writer) +void CentersTableBuilder::Freeze(Writer & writer) const { CentersTableV0::Header header; diff --git a/indexer/centers_table.hpp b/indexer/centers_table.hpp index a6ff65d4dd..8166352ada 100644 --- a/indexer/centers_table.hpp +++ b/indexer/centers_table.hpp @@ -68,7 +68,7 @@ public: } void Put(uint32_t featureId, m2::PointD const & center); - void Freeze(Writer & writer); + void Freeze(Writer & writer) const; private: serial::CodingParams m_codingParams; diff --git a/search/lazy_centers_table.hpp b/search/lazy_centers_table.hpp index 68dc97146a..2ab50bfff4 100644 --- a/search/lazy_centers_table.hpp +++ b/search/lazy_centers_table.hpp @@ -22,7 +22,7 @@ public: STATE_FAILED }; - LazyCentersTable(MwmValue & value); + explicit LazyCentersTable(MwmValue & value); inline State GetState() const { return m_state; } diff --git a/search/mwm_context.hpp b/search/mwm_context.hpp index 1c1382cb5b..a146c1098c 100644 --- a/search/mwm_context.hpp +++ b/search/mwm_context.hpp @@ -9,6 +9,8 @@ #include "base/macros.hpp" +#include "std/shared_ptr.hpp" +#include "std/string.hpp" #include "std/unique_ptr.hpp" class MwmValue; diff --git a/search/pre_ranker.cpp b/search/pre_ranker.cpp index 285f0c037c..d0d985bb9e 100644 --- a/search/pre_ranker.cpp +++ b/search/pre_ranker.cpp @@ -17,8 +17,6 @@ namespace search { namespace { -size_t const kBatchSize = 100; - struct LessFeatureID { using TValue = PreResult1; @@ -68,7 +66,7 @@ void PreRanker::FillMissingFieldsInPreResults() unique_ptr ranks = make_unique(); unique_ptr centers; - bool const fillCenters = (Size() > kBatchSize); + bool const fillCenters = (Size() > BatchSize()); if (fillCenters) m_pivotFeatures.SetPosition(m_params.m_accuratePivotCenter, m_params.m_scale); @@ -120,7 +118,7 @@ void PreRanker::Filter(bool viewportSearch) sort(m_results.begin(), m_results.end(), &PreResult1::LessDistance); - if (m_results.size() > kBatchSize) + if (m_results.size() > BatchSize()) { // Priority is some kind of distance from the viewport or // position, therefore if we have a bunch of results with the same @@ -129,15 +127,15 @@ void PreRanker::Filter(bool viewportSearch) // feature id) this code randomly selects tail of the // sorted-by-priority list of pre-results. - double const last = m_results[kBatchSize - 1].GetDistance(); + double const last = m_results[BatchSize()].GetDistance(); - auto b = m_results.begin() + kBatchSize - 1; + auto b = m_results.begin() + BatchSize(); for (; b != m_results.begin() && b->GetDistance() == last; --b) ; if (b->GetDistance() != last) ++b; - auto e = m_results.begin() + kBatchSize; + auto e = m_results.begin() + BatchSize(); for (; e != m_results.end() && e->GetDistance() == last; ++e) ; @@ -153,11 +151,11 @@ void PreRanker::Filter(bool viewportSearch) minstd_rand engine; shuffle(b, e, engine); } - filtered.insert(m_results.begin(), m_results.begin() + min(m_results.size(), kBatchSize)); + filtered.insert(m_results.begin(), m_results.begin() + min(m_results.size(), BatchSize())); if (!viewportSearch) { - size_t n = min(m_results.size(), kBatchSize); + size_t n = min(m_results.size(), BatchSize()); nth_element(m_results.begin(), m_results.begin() + n, m_results.end(), &PreResult1::LessRank); filtered.insert(m_results.begin(), m_results.begin() + n); } diff --git a/search/pre_ranker.hpp b/search/pre_ranker.hpp index 6960485137..aa2622ef27 100644 --- a/search/pre_ranker.hpp +++ b/search/pre_ranker.hpp @@ -32,6 +32,8 @@ public: // compute the distance from a feature to the pivot. m2::PointD m_accuratePivotCenter = m2::PointD(0, 0); int m_scale = 0; + + size_t m_batchSize = 100; }; PreRanker(Index const & index, Ranker & ranker, size_t limit); @@ -39,10 +41,6 @@ public: void Init(Params const & params); inline void SetViewportSearch(bool viewportSearch) { m_viewportSearch = viewportSearch; } - inline void SetAccuratePivotCenter(m2::PointD const & center) - { - m_params.m_accuratePivotCenter = center; - } template void Emplace(TArgs &&... args) @@ -63,6 +61,7 @@ public: void UpdateResults(bool lastUpdate); inline size_t Size() const { return m_results.size(); } + inline size_t BatchSize() const { return m_params.m_batchSize; } inline size_t NumSentResults() const { return m_numSentResults; } inline size_t Limit() const { return m_limit; } diff --git a/search/ranker.hpp b/search/ranker.hpp index e13f2fecb3..fbea55b50c 100644 --- a/search/ranker.hpp +++ b/search/ranker.hpp @@ -70,14 +70,10 @@ public: Ranker(Index const & index, storage::CountryInfoGetter const & infoGetter, CategoriesHolder const & categories, vector const & suggests, my::Cancellable const & cancellable); + virtual ~Ranker() = default; void Init(Params const & params, Geocoder::Params const & geocoderParams); - inline void SetAccuratePivotCenter(m2::PointD const & center) - { - m_params.m_accuratePivotCenter = center; - } - bool IsResultExists(PreResult2 const & p, vector const & values); void MakePreResult2(Geocoder::Params const & params, vector & cont, @@ -93,11 +89,12 @@ public: void GetBestMatchName(FeatureType const & f, string & name) const; void ProcessSuggestions(vector & vec, Results & res) const; - Results & GetResults() { return m_results; } - void UpdateResults(bool lastUpdate); + virtual void SetPreResults1(vector && preResults1) { m_preResults1 = move(preResults1); } + virtual void UpdateResults(bool lastUpdate); + + inline Results & GetResults() { return m_results; } inline void FlushResults() { UpdateResults(true /* lastUpdate */); } - void SetPreResults1(vector && preResults1) { m_preResults1 = move(preResults1); } void ClearCaches(); inline void SetLocalityFinderLanguage(int8_t code) { m_locality.SetLanguage(code); } diff --git a/search/search_integration_tests/pre_ranker_test.cpp b/search/search_integration_tests/pre_ranker_test.cpp new file mode 100644 index 0000000000..50eb2f5502 --- /dev/null +++ b/search/search_integration_tests/pre_ranker_test.cpp @@ -0,0 +1,171 @@ +#include "testing/testing.hpp" + +#include "search/intermediate_result.hpp" +#include "search/model.hpp" +#include "search/pre_ranker.hpp" +#include "search/ranker.hpp" +#include "search/search_integration_tests/helpers.hpp" +#include "search/search_tests_support/test_search_engine.hpp" +#include "search/suggest.hpp" + +#include "indexer/categories_holder.hpp" +#include "indexer/feature_algo.hpp" +#include "indexer/features_vector.hpp" +#include "indexer/mwm_set.hpp" +#include "indexer/scales.hpp" + +#include "generator/generator_tests_support/test_feature.hpp" +#include "generator/generator_tests_support/test_mwm_builder.hpp" + +#include "geometry/mercator.hpp" + +#include "platform/country_defines.hpp" +#include "platform/local_country_file.hpp" + +#include "base/assert.hpp" +#include "base/cancellable.hpp" +#include "base/limited_priority_queue.hpp" +#include "base/math.hpp" +#include "base/stl_add.hpp" + +#include "std/algorithm.hpp" +#include "std/iterator.hpp" +#include "std/vector.hpp" + +using namespace generator::tests_support; +using namespace search::tests_support; + +namespace search +{ +namespace +{ +// Returns the boundary such that at least |size| elements from +// |distance| are less than or equal to the boundary. +double GetBoundary(vector const & distances, size_t size) +{ + CHECK_LESS_OR_EQUAL(size, distances.size(), ()); + my::limited_priority_queue queue(size); + for (auto const & distance : distances) + queue.push(distance); + return queue.empty() ? 0.0 : queue.top(); +} + +class TestRanker : public Ranker +{ +public: + TestRanker(TestSearchEngine & engine, vector const & suggests, + my::Cancellable const & cancellable, vector & results) + : Ranker(static_cast(engine), engine.GetCountryInfoGetter(), + GetDefaultCategories(), suggests, cancellable) + , m_results(results) + { + } + + inline bool Finished() const { return m_finished; } + + // Ranker overrides: + void SetPreResults1(vector && preResults1) override + { + CHECK(!Finished(), ()); + move(preResults1.begin(), preResults1.end(), back_inserter(m_results)); + preResults1.clear(); + } + + void UpdateResults(bool lastUpdate) override + { + CHECK(!Finished(), ()); + if (lastUpdate) + m_finished = true; + } + +private: + vector & m_results; + bool m_finished = false; +}; + +class PreRankerTest : public SearchTest +{ +public: + vector m_suggests; + my::Cancellable m_cancellable; +}; + +UNIT_CLASS_TEST(PreRankerTest, Smoke) +{ + // Tests that PreRanker correctly computes distances to pivot when + // number of results is larger than batch size, and that PreRanker + // emits results nearest to the pivot. + + m2::PointD const kPivot(0.5, 0.5); + size_t const kBatchSize = 50; + + vector pois; + + for (int x = -5; x <= 5; ++x) + { + for (int y = -5; y <= 5; ++y) + { + pois.emplace_back(m2::PointD(x, y), "cafe", "en"); + pois.back().SetTypes({{"amenity", "cafe"}}); + } + } + + TEST_LESS(kBatchSize, pois.size(), ()); + + auto mwmId = BuildCountry("Cafeland", [&](TestMwmBuilder & builder) { + for (auto const & poi : pois) + builder.Add(poi); + }); + + vector results; + TestRanker ranker(m_engine, m_suggests, m_cancellable, results); + + PreRanker preRanker(m_engine, ranker, pois.size()); + PreRanker::Params params; + params.m_accuratePivotCenter = kPivot; + params.m_scale = scales::GetUpperScale(); + params.m_batchSize = kBatchSize; + preRanker.Init(params); + preRanker.SetViewportSearch(true); + + vector distances(pois.size()); + vector emit(pois.size()); + + FeaturesVectorTest fv(mwmId.GetInfo()->GetLocalFile().GetPath(MapOptions::Map)); + fv.GetVector().ForEach([&](FeatureType & ft, uint32_t index) { + FeatureID id(mwmId, index); + + PreRankingInfo info; + info.m_startToken = 0; + info.m_endToken = 1; + info.m_searchType = SearchModel::SEARCH_TYPE_POI; + preRanker.Emplace(id, info); + + TEST_LESS(index, pois.size(), ()); + distances[index] = MercatorBounds::DistanceOnEarth(feature::GetCenter(ft), kPivot); + emit[index] = true; + }); + + preRanker.UpdateResults(true /* lastUpdate */); + + TEST(all_of(emit.begin(), emit.end(), IdFunctor()), (emit)); + TEST(ranker.Finished(), ()); + TEST_EQUAL(results.size(), kBatchSize, ()); + + double const boundary = GetBoundary(distances, kBatchSize); + + vector checked(pois.size()); + for (size_t i = 0; i < results.size(); ++i) + { + size_t const index = results[i].GetId().m_index; + TEST_LESS(index, pois.size(), ()); + + TEST(!checked[index], (index)); + TEST_LESS_OR_EQUAL(results[i].GetDistance(), boundary, ()); + TEST(my::AlmostEqualAbs(distances[index], results[i].GetDistance(), 1e-3), + (distances[index], results[i].GetDistance())); + checked[index] = true; + } +} +} // namespace +} // namespace search diff --git a/search/search_integration_tests/search_integration_tests.pro b/search/search_integration_tests/search_integration_tests.pro index 2ede061546..c764ff9498 100644 --- a/search/search_integration_tests/search_integration_tests.pro +++ b/search/search_integration_tests/search_integration_tests.pro @@ -21,6 +21,7 @@ SOURCES += \ ../../testing/testingmain.cpp \ generate_tests.cpp \ helpers.cpp \ + pre_ranker_test.cpp \ processor_test.cpp \ smoke_test.cpp \ diff --git a/tools/unix/generate_mwm.sh b/tools/unix/generate_mwm.sh index 7be2d99dde..cededb574e 100755 --- a/tools/unix/generate_mwm.sh +++ b/tools/unix/generate_mwm.sh @@ -77,7 +77,7 @@ trap "rm -rf \"${INTDIR}\"" EXIT SIGINT SIGTERM # Create MWM file INTDIR_FLAG="--intermediate_data_path=$INTDIR/ --node_storage=map" -GENERATE_EVERYTHING='--generate_features=true --generate_geometry=true --generate_index=true --generate_search_index=true --generate_centers_table=true' +GENERATE_EVERYTHING='--generate_features=true --generate_geometry=true --generate_index=true --generate_search_index=true' COASTS="${COASTS-WorldCoasts.geom}" if [ -f "$COASTS" ]; then if [ ! -f "$TBORDERS/$BASE_NAME.poly" ]; then diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 51337618c1..b7aeadccee 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -401,13 +401,13 @@ if [ "$MODE" == "mwm" ]; then if [ -n "$OPT_WORLD" ]; then ( "$GENERATOR_TOOL" $PARAMS --planet_version="$MWM_VERSION" --output=World 2>> "$LOG_PATH/World.log" - "$GENERATOR_TOOL" --data_path="$TARGET" --planet_version="$MWM_VERSION" --user_resource_path="$DATA_PATH/" -generate_search_index -generate_centers_table --output=World 2>> "$LOG_PATH/World.log" + "$GENERATOR_TOOL" --data_path="$TARGET" --planet_version="$MWM_VERSION" --user_resource_path="$DATA_PATH/" -generate_search_index --output=World 2>> "$LOG_PATH/World.log" ) & "$GENERATOR_TOOL" $PARAMS --planet_version="$MWM_VERSION" --output=WorldCoasts 2>> "$LOG_PATH/WorldCoasts.log" & fi if [ -z "$NO_REGIONS" ]; then - PARAMS_WITH_SEARCH="$PARAMS -generate_search_index -generate_centers_table" + PARAMS_WITH_SEARCH="$PARAMS -generate_search_index" for file in "$INTDIR"/tmp/*.mwm.tmp; do if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then BASENAME="$(basename "$file" .mwm.tmp)"