diff --git a/generator/generator_tests_support/test_mwm_builder.cpp b/generator/generator_tests_support/test_mwm_builder.cpp index eb8717fa80..99c6f374f3 100644 --- a/generator/generator_tests_support/test_mwm_builder.cpp +++ b/generator/generator_tests_support/test_mwm_builder.cpp @@ -131,7 +131,7 @@ void TestMwmBuilder::Finish() CHECK(indexer::BuildIndexFromDataFile(path, path), ("Can't build geometry index.")); - CHECK(indexer::BuildSearchIndexFromDataFile(path, true /* forceRebuild */), + CHECK(indexer::BuildSearchIndexFromDataFile(1 /* threadsCount */, path, true /* forceRebuild */), ("Can't build search index.")); if (m_type == feature::DataHeader::world) diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index aebcd74bb1..e3983e4aaa 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -382,7 +382,8 @@ int main(int argc, char ** argv) { LOG(LINFO, ("Generating search index for", datFile)); - if (!indexer::BuildSearchIndexFromDataFile(datFile, true)) + /// @todo Make threads count according to environment (single mwm build or planet build). + if (!indexer::BuildSearchIndexFromDataFile(8, datFile, true)) LOG(LCRITICAL, ("Error generating search index.")); LOG(LINFO, ("Generating rank table for", datFile)); diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index 6578e08bbb..533230ff3b 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -38,10 +38,11 @@ #include #include -#include -#include +#include +#include #include #include +#include using namespace std; @@ -318,26 +319,47 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest const & features, synonyms.get(), keyValuePairs, categoriesHolder, header.GetScaleRange(), valueBuilder)); } -void BuildAddressTable(FilesContainerR & container, Writer & writer) +pair GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, + string const & streetName) +{ + size_t streetIndex = 0; + strings::UniString const street = search::GetStreetNameAsKey(streetName); + + bool const hasStreet = !street.empty(); + if (hasStreet) + { + FeatureType ft; + VERIFY(ctx.GetFeature(featureID, ft), ()); + + using TStreet = search::ReverseGeocoder::Street; + vector streets; + search::ReverseGeocoder::GetNearbyStreets(ctx, feature::GetCenter(ft), streets); + + streetIndex = search::ReverseGeocoder::GetMatchedStreetIndex(street, streets); + if (streetIndex < streets.size()) + return { base::checked_cast(streetIndex), true }; + } + + return { hasStreet ? 1 : 0, false }; +} + +void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Writer & writer) { // Read all street names to memory. ReaderSource src(container.GetReader(SEARCH_TOKENS_FILE_TAG)); vector addrs; + while (src.Size() > 0) { - while (src.Size() > 0) - { - addrs.push_back({}); - addrs.back().Deserialize(src); - } + addrs.push_back({}); + addrs.back().Deserialize(src); } - uint32_t const featuresCount = static_cast(addrs.size()); + uint32_t const featuresCount = base::checked_cast(addrs.size()); // Initialize temporary source for the current mwm file. FrozenDataSource dataSource; auto const res = dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName())); ASSERT_EQUAL(res.second, MwmSet::RegResult::Success, ()); - uint32_t const threadsCount = 8; vector> contexts(threadsCount); uint32_t address = 0, missing = 0; @@ -351,50 +373,28 @@ void BuildAddressTable(FilesContainerR & container, Writer & writer) // Thread working function. auto const fn = [&](uint32_t threadIdx) { - uint32_t const count = (featuresCount + threadsCount - 1) / threadsCount; - uint32_t const beg = count * threadIdx; - uint32_t const end = std::min(beg + count, featuresCount); + uint64_t const fc = featuresCount; + uint32_t const beg = static_cast(fc * threadIdx / threadsCount); + uint32_t const end = static_cast(fc * (threadIdx + 1) / threadsCount); for (uint32_t i = beg; i < end; ++i) { - size_t streetIndex = 0; - bool streetMatched = false; - strings::UniString const street = search::GetStreetNameAsKey(addrs[i].Get(feature::AddressData::STREET)); - - bool const hasStreet = !street.empty(); - if (hasStreet) - { - FeatureType ft; - VERIFY(contexts[threadIdx]->GetFeature(i, ft), ()); - - using TStreet = search::ReverseGeocoder::Street; - vector streets; - search::ReverseGeocoder::GetNearbyStreets(*(contexts[threadIdx]), feature::GetCenter(ft), streets); - - streetIndex = search::ReverseGeocoder::GetMatchedStreetIndex(street, streets); - if (streetIndex < streets.size()) - streetMatched = true; - } + auto const res = GetStreetIndex(*(contexts[threadIdx]), i, addrs[i].Get(feature::AddressData::STREET)); lock_guard guard(resMutex); - if (streetMatched) + if (res.second) { - results[i] = static_cast(streetIndex); - ++bounds[streetIndex]; + results[i] = res.first; + ++bounds[res.first]; ++address; } - else + else if (res.first > 0) { - if (hasStreet) - { ++missing; ++address; - } } } - - LOG(LINFO, ("Thread finished, idx = ", threadIdx)); }; // Prepare threads and mwm contexts for each thread. @@ -402,7 +402,7 @@ void BuildAddressTable(FilesContainerR & container, Writer & writer) for (size_t i = 0; i < threadsCount; ++i) { auto handle = dataSource.GetMwmHandleById(res.first); - contexts[i] = make_unique(std::move(handle)); + contexts[i] = make_unique(move(handle)); threads.emplace_back(fn, i); } @@ -434,7 +434,7 @@ void BuildAddressTable(FilesContainerR & container, Writer & writer) namespace indexer { -bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild) +bool BuildSearchIndexFromDataFile(uint32_t threadsCount, string const & filename, bool forceRebuild) { Platform & platform = GetPlatform(); @@ -457,7 +457,7 @@ bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild) if (filename != WORLD_FILE_NAME && filename != WORLD_COASTS_FILE_NAME) { FileWriter writer(addrFilePath); - BuildAddressTable(readContainer, writer); + BuildAddressTable(threadsCount, readContainer, writer); LOG(LINFO, ("Search address table size =", writer.Size())); } { diff --git a/generator/search_index_builder.hpp b/generator/search_index_builder.hpp index c9cc1bbdca..351f75bbe3 100644 --- a/generator/search_index_builder.hpp +++ b/generator/search_index_builder.hpp @@ -11,7 +11,7 @@ namespace indexer // An attempt to rewrite the search index of an old mwm may result in a future crash // when using search because this function does not update mwm's version. This results // in version mismatch when trying to read the index. -bool BuildSearchIndexFromDataFile(std::string const & filename, bool forceRebuild = false); +bool BuildSearchIndexFromDataFile(uint32_t threadsCount, std::string const & filename, bool forceRebuild = false); void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter); } // namespace indexer diff --git a/search/reverse_geocoder.cpp b/search/reverse_geocoder.cpp index f34fa19ac6..300e8977ee 100644 --- a/search/reverse_geocoder.cpp +++ b/search/reverse_geocoder.cpp @@ -26,6 +26,7 @@ size_t constexpr kMaxNumTriesToApproxAddress = 10; ReverseGeocoder::ReverseGeocoder(DataSource const & dataSource) : m_dataSource(dataSource) {} +// static void ReverseGeocoder::GetNearbyStreets(search::MwmContext & context, m2::PointD const & center, vector & streets) { @@ -53,7 +54,7 @@ void ReverseGeocoder::GetNearbyStreets(search::MwmContext & context, m2::PointD } void ReverseGeocoder::GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD const & center, - vector & streets) const + vector & streets) const { MwmSet::MwmHandle mwmHandle = m_dataSource.GetMwmHandleById(id); if (mwmHandle.IsAlive()) diff --git a/search/reverse_geocoder.hpp b/search/reverse_geocoder.hpp index 72c380449f..b69dc36fa1 100644 --- a/search/reverse_geocoder.hpp +++ b/search/reverse_geocoder.hpp @@ -79,8 +79,7 @@ public: /// @return Sorted by distance streets vector for the specified MwmId. //@{ static void GetNearbyStreets(search::MwmContext & context, m2::PointD const & center, - vector & streets); - + vector & streets); void GetNearbyStreets(MwmSet::MwmId const & id, m2::PointD const & center, vector & streets) const; void GetNearbyStreets(FeatureType & ft, vector & streets) const;