From d39089f03f72e2ddfc83ad656a33f729267e814f Mon Sep 17 00:00:00 2001 From: Maxim Pimenov Date: Wed, 25 Jul 2018 17:29:46 +0300 Subject: [PATCH] [generator] [search] Minor fixes. --- .../test_mwm_builder.cpp | 2 +- generator/generator_tool/generator_tool.cpp | 3 +- generator/osm_xml_source.hpp | 103 ++++++++++-------- generator/search_index_builder.cpp | 51 +++++---- generator/search_index_builder.hpp | 3 +- 5 files changed, 91 insertions(+), 71 deletions(-) diff --git a/generator/generator_tests_support/test_mwm_builder.cpp b/generator/generator_tests_support/test_mwm_builder.cpp index 99c6f374f3..16a216f2f1 100644 --- a/generator/generator_tests_support/test_mwm_builder.cpp +++ b/generator/generator_tests_support/test_mwm_builder.cpp @@ -131,7 +131,7 @@ void TestMwmBuilder::Finish() CHECK(indexer::BuildIndexFromDataFile(path, path), ("Can't build geometry index.")); - CHECK(indexer::BuildSearchIndexFromDataFile(1 /* threadsCount */, path, true /* forceRebuild */), + CHECK(indexer::BuildSearchIndexFromDataFile(path, true /* forceRebuild */, 1 /* threadsCount */), ("Can't build search index.")); if (m_type == feature::DataHeader::world) diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index e3983e4aaa..d131434e4a 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -383,7 +383,8 @@ int main(int argc, char ** argv) LOG(LINFO, ("Generating search index for", datFile)); /// @todo Make threads count according to environment (single mwm build or planet build). - if (!indexer::BuildSearchIndexFromDataFile(8, datFile, true)) + if (!indexer::BuildSearchIndexFromDataFile(datFile, true /* forceRebuild */, + 1 /* threadsCount */)) LOG(LCRITICAL, ("Error generating search index.")); LOG(LINFO, ("Generating rank table for", datFile)); diff --git a/generator/osm_xml_source.hpp b/generator/osm_xml_source.hpp index 79a24cb2cc..4e72361e68 100644 --- a/generator/osm_xml_source.hpp +++ b/generator/osm_xml_source.hpp @@ -2,19 +2,18 @@ #include "generator/osm_element.hpp" +#include "base/assert.hpp" +#include "base/string_utils.hpp" + +#include +#include + class XMLSource { - OsmElement m_parent; - OsmElement m_child; - - size_t m_depth = 0; - OsmElement * m_current = nullptr; - - using TEmmiterFn = function; - TEmmiterFn m_EmmiterFn; - public: - XMLSource(TEmmiterFn fn) : m_EmmiterFn(fn) {} + using Emitter = std::function; + + XMLSource(Emitter fn) : m_emitter(fn) {} void CharData(std::string const &) {} @@ -24,13 +23,13 @@ public: return; if (key == "id") - CHECK ( strings::to_uint64(value, m_current->id), ("Unknown element with invalid id : ", value) ); + CHECK(strings::to_uint64(value, m_current->id), ("Unknown element with invalid id:", value)); else if (key == "lon") - CHECK ( strings::to_double(value, m_current->lon), ("Bad node lon : ", value) ); + CHECK(strings::to_double(value, m_current->lon), ("Bad node lon:", value)); else if (key == "lat") - CHECK ( strings::to_double(value, m_current->lat), ("Bad node lat : ", value) ); + CHECK(strings::to_double(value, m_current->lat), ("Bad node lat:", value)); else if (key == "ref") - CHECK ( strings::to_uint64(value, m_current->ref), ("Bad node ref in way : ", value) ); + CHECK(strings::to_uint64(value, m_current->ref), ("Bad node ref in way:", value)); else if (key == "k") m_current->k = value; else if (key == "v") @@ -46,20 +45,22 @@ public: ASSERT_GREATER_OR_EQUAL(tagName.size(), 2, ()); // As tagKey we use first two char of tag name. - OsmElement::EntityType tagKey = OsmElement::EntityType(*reinterpret_cast(tagName.data())); + OsmElement::EntityType tagKey = + OsmElement::EntityType(*reinterpret_cast(tagName.data())); switch (++m_depth) { - case 1: - m_current = nullptr; - break; - case 2: - m_current = &m_parent; - m_current->type = tagKey; - break; - default: - m_current = &m_child; - m_current->type = tagKey; + case 1: + m_current = nullptr; + break; + case 2: + m_current = &m_parent; + m_current->type = tagKey; + break; + default: + m_current = &m_child; + m_current->type = tagKey; + break; } return true; } @@ -68,30 +69,38 @@ public: { switch (--m_depth) { - case 0: - break; + case 0: + break; - case 1: - m_EmmiterFn(m_current); - m_parent.Clear(); - break; + case 1: + m_emitter(m_current); + m_parent.Clear(); + break; - default: - switch (m_child.type) - { - case OsmElement::EntityType::Member: - m_parent.AddMember(m_child.ref, m_child.memberType, m_child.role); - break; - case OsmElement::EntityType::Tag: - m_parent.AddTag(m_child.k, m_child.v); - break; - case OsmElement::EntityType::Nd: - m_parent.AddNd(m_child.ref); - default: - break; - } - m_current = &m_parent; - m_child.Clear(); + default: + switch (m_child.type) + { + case OsmElement::EntityType::Member: + m_parent.AddMember(m_child.ref, m_child.memberType, m_child.role); + break; + case OsmElement::EntityType::Tag: + m_parent.AddTag(m_child.k, m_child.v); + break; + case OsmElement::EntityType::Nd: + m_parent.AddNd(m_child.ref); + break; + } + m_current = &m_parent; + m_child.Clear(); } } + +private: + OsmElement m_parent; + OsmElement m_child; + + size_t m_depth = 0; + OsmElement * m_current = nullptr; + + Emitter m_emitter; }; diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index 533230ff3b..e53cc6c5a1 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -48,7 +48,6 @@ using namespace std; #define SYNONYMS_FILE "synonyms.txt" - namespace { class SynonymsHolder @@ -319,8 +318,8 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest const & features, synonyms.get(), keyValuePairs, categoriesHolder, header.GetScaleRange(), valueBuilder)); } -pair GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, - string const & streetName) +bool GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, string const & streetName, + uint32_t & result) { size_t streetIndex = 0; strings::UniString const street = search::GetStreetNameAsKey(streetName); @@ -337,13 +336,17 @@ pair GetStreetIndex(search::MwmContext & ctx, uint32_t featureID streetIndex = search::ReverseGeocoder::GetMatchedStreetIndex(street, streets); if (streetIndex < streets.size()) - return { base::checked_cast(streetIndex), true }; + { + result = base::checked_cast(streetIndex); + return true; + } } - return { hasStreet ? 1 : 0, false }; + result = hasStreet ? 1 : 0; + return false; } -void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Writer & writer) +void BuildAddressTable(FilesContainerR & container, Writer & writer, uint32_t threadsCount) { // Read all street names to memory. ReaderSource src(container.GetReader(SEARCH_TOKENS_FILE_TAG)); @@ -357,8 +360,13 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write // Initialize temporary source for the current mwm file. FrozenDataSource dataSource; - auto const res = dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName())); - ASSERT_EQUAL(res.second, MwmSet::RegResult::Success, ()); + MwmSet::MwmId mwmId; + { + auto const regResult = + dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName())); + ASSERT_EQUAL(regResult.second, MwmSet::RegResult::Success, ()); + mwmId = regResult.first; + } vector> contexts(threadsCount); @@ -371,28 +379,29 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write mutex resMutex; // Thread working function. - auto const fn = [&](uint32_t threadIdx) - { - uint64_t const fc = featuresCount; + auto const fn = [&](uint32_t threadIdx) { + uint64_t const fc = static_cast(featuresCount); uint32_t const beg = static_cast(fc * threadIdx / threadsCount); uint32_t const end = static_cast(fc * (threadIdx + 1) / threadsCount); for (uint32_t i = beg; i < end; ++i) { - auto const res = GetStreetIndex(*(contexts[threadIdx]), i, addrs[i].Get(feature::AddressData::STREET)); + uint32_t streetIndex; + bool const found = GetStreetIndex(*(contexts[threadIdx]), i, + addrs[i].Get(feature::AddressData::STREET), streetIndex); lock_guard guard(resMutex); - if (res.second) + if (found) { - results[i] = res.first; - ++bounds[res.first]; + results[i] = streetIndex; + ++bounds[streetIndex]; ++address; } - else if (res.first > 0) + else if (streetIndex > 0) { - ++missing; - ++address; + ++missing; + ++address; } } }; @@ -401,7 +410,7 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write vector threads; for (size_t i = 0; i < threadsCount; ++i) { - auto handle = dataSource.GetMwmHandleById(res.first); + auto handle = dataSource.GetMwmHandleById(mwmId); contexts[i] = make_unique(move(handle)); threads.emplace_back(fn, i); } @@ -434,7 +443,7 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write namespace indexer { -bool BuildSearchIndexFromDataFile(uint32_t threadsCount, string const & filename, bool forceRebuild) +bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild, uint32_t threadsCount) { Platform & platform = GetPlatform(); @@ -457,7 +466,7 @@ bool BuildSearchIndexFromDataFile(uint32_t threadsCount, string const & filename if (filename != WORLD_FILE_NAME && filename != WORLD_COASTS_FILE_NAME) { FileWriter writer(addrFilePath); - BuildAddressTable(threadsCount, readContainer, writer); + BuildAddressTable(readContainer, writer, threadsCount); LOG(LINFO, ("Search address table size =", writer.Size())); } { diff --git a/generator/search_index_builder.hpp b/generator/search_index_builder.hpp index 351f75bbe3..7ea1f6ff9d 100644 --- a/generator/search_index_builder.hpp +++ b/generator/search_index_builder.hpp @@ -11,7 +11,8 @@ namespace indexer // An attempt to rewrite the search index of an old mwm may result in a future crash // when using search because this function does not update mwm's version. This results // in version mismatch when trying to read the index. -bool BuildSearchIndexFromDataFile(uint32_t threadsCount, std::string const & filename, bool forceRebuild = false); +bool BuildSearchIndexFromDataFile(std::string const & filename, bool forceRebuild, + uint32_t threadsCount); void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter); } // namespace indexer