From 8611a19c0315cf35fe745209abc2169fc6dda676 Mon Sep 17 00:00:00 2001 From: vng Date: Sun, 22 Jul 2018 10:17:18 +0300 Subject: [PATCH] [generator] Speed up search address section building. --- generator/search_index_builder.cpp | 106 +++++++++++++++++++++-------- search/reverse_geocoder.cpp | 15 ++-- search/reverse_geocoder.hpp | 4 ++ 3 files changed, 94 insertions(+), 31 deletions(-) diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index ca256babba..6578e08bbb 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -1,6 +1,7 @@ #include "search_index_builder.hpp" #include "search/common.hpp" +#include "search/mwm_context.hpp" #include "search/reverse_geocoder.hpp" #include "search/search_index_values.hpp" #include "search/search_trie.hpp" @@ -319,54 +320,105 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest const & features, void BuildAddressTable(FilesContainerR & container, Writer & writer) { - ReaderSource src = container.GetReader(SEARCH_TOKENS_FILE_TAG); + // Read all street names to memory. + ReaderSource src(container.GetReader(SEARCH_TOKENS_FILE_TAG)); + vector addrs; + { + while (src.Size() > 0) + { + addrs.push_back({}); + addrs.back().Deserialize(src); + } + } + uint32_t const featuresCount = static_cast(addrs.size()); + + // Initialize temporary source for the current mwm file. + FrozenDataSource dataSource; + auto const res = dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName())); + ASSERT_EQUAL(res.second, MwmSet::RegResult::Success, ()); + + uint32_t const threadsCount = 8; + vector> contexts(threadsCount); + uint32_t address = 0, missing = 0; map bounds; - FrozenDataSource dataSource; - /// @ todo Make some better solution, or legalize MakeTemporary. - auto const res = dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName())); - ASSERT_EQUAL(res.second, MwmSet::RegResult::Success, ()); - search::ReverseGeocoder rgc(dataSource); + uint32_t const kEmptyResult = uint32_t(-1); + vector results(featuresCount, kEmptyResult); + mutex resMutex; + + // Thread working function. + auto const fn = [&](uint32_t threadIdx) { - FixedBitsDDVector<3, FileReader>::Builder building2Street(writer); + uint32_t const count = (featuresCount + threadsCount - 1) / threadsCount; + uint32_t const beg = count * threadIdx; + uint32_t const end = std::min(beg + count, featuresCount); - FeaturesVectorTest features(container); - for (uint32_t index = 0; src.Size() > 0; ++index) + for (uint32_t i = beg; i < end; ++i) { - feature::AddressData data; - data.Deserialize(src); - size_t streetIndex = 0; bool streetMatched = false; - strings::UniString const street = search::GetStreetNameAsKey(data.Get(feature::AddressData::STREET)); - if (!street.empty()) + strings::UniString const street = search::GetStreetNameAsKey(addrs[i].Get(feature::AddressData::STREET)); + + bool const hasStreet = !street.empty(); + if (hasStreet) { FeatureType ft; - features.GetVector().GetByIndex(index, ft); - ft.SetID({res.first, index}); + VERIFY(contexts[threadIdx]->GetFeature(i, ft), ()); using TStreet = search::ReverseGeocoder::Street; vector streets; - rgc.GetNearbyStreets(ft, streets); + search::ReverseGeocoder::GetNearbyStreets(*(contexts[threadIdx]), feature::GetCenter(ft), streets); - streetIndex = rgc.GetMatchedStreetIndex(street, streets); + streetIndex = search::ReverseGeocoder::GetMatchedStreetIndex(street, streets); if (streetIndex < streets.size()) - { - ++bounds[streetIndex]; streetMatched = true; - } - else - { - ++missing; - } + } + + lock_guard guard(resMutex); + + if (streetMatched) + { + results[i] = static_cast(streetIndex); + ++bounds[streetIndex]; ++address; } - if (streetMatched) - building2Street.PushBack(base::checked_cast(streetIndex)); else + { + if (hasStreet) + { + ++missing; + ++address; + } + } + } + + LOG(LINFO, ("Thread finished, idx = ", threadIdx)); + }; + + // Prepare threads and mwm contexts for each thread. + vector threads; + for (size_t i = 0; i < threadsCount; ++i) + { + auto handle = dataSource.GetMwmHandleById(res.first); + contexts[i] = make_unique(std::move(handle)); + threads.emplace_back(fn, i); + } + + // Wait for thread's finish. + for (auto & t : threads) + t.join(); + + // Flush results to disk. + { + FixedBitsDDVector<3, FileReader>::Builder building2Street(writer); + for (auto i : results) + { + if (i == kEmptyResult) building2Street.PushBackUndefined(); + else + building2Street.PushBack(i); } LOG(LINFO, ("Address: Building -> Street (opt, all)", building2Street.GetCount())); diff --git a/search/reverse_geocoder.cpp b/search/reverse_geocoder.cpp index 6b521ebe9f..f34fa19ac6 100644 --- a/search/reverse_geocoder.cpp +++ b/search/reverse_geocoder.cpp @@ -26,8 +26,8 @@ size_t constexpr kMaxNumTriesToApproxAddress = 10; ReverseGeocoder::ReverseGeocoder(DataSource const & dataSource) : m_dataSource(dataSource) {} -void ReverseGeocoder::GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD const & center, - vector & streets) const +void ReverseGeocoder::GetNearbyStreets(search::MwmContext & context, m2::PointD const & center, + vector & streets) { m2::RectD const rect = GetLookupRect(center, kLookupRadiusM); @@ -48,11 +48,18 @@ void ReverseGeocoder::GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD cons streets.emplace_back(ft.GetID(), feature::GetMinDistanceMeters(ft, center), name); }; + context.ForEachFeature(rect, addStreet); + sort(streets.begin(), streets.end(), my::LessBy(&Street::m_distanceMeters)); +} + +void ReverseGeocoder::GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD const & center, + vector & streets) const +{ MwmSet::MwmHandle mwmHandle = m_dataSource.GetMwmHandleById(id); if (mwmHandle.IsAlive()) { - search::MwmContext(move(mwmHandle)).ForEachFeature(rect, addStreet); - sort(streets.begin(), streets.end(), my::LessBy(&Street::m_distanceMeters)); + search::MwmContext context(move(mwmHandle)); + GetNearbyStreets(context, center, streets); } } diff --git a/search/reverse_geocoder.hpp b/search/reverse_geocoder.hpp index 6b2fd65ac9..72c380449f 100644 --- a/search/reverse_geocoder.hpp +++ b/search/reverse_geocoder.hpp @@ -16,6 +16,7 @@ class DataSource; namespace search { +class MwmContext; class ReverseGeocoder { @@ -77,6 +78,9 @@ public: /// @return Sorted by distance streets vector for the specified MwmId. //@{ + static void GetNearbyStreets(search::MwmContext & context, m2::PointD const & center, + vector & streets); + void GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD const & center, vector & streets) const; void GetNearbyStreets(FeatureType & ft, vector & streets) const;