From 23c9c27f596db2aa014fa39d58458613df83a067 Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Fri, 15 Nov 2019 16:09:47 +0300 Subject: [PATCH] [search] Use postcodes from temp address section while building search index. Prepare to remove postcodes from metadata. --- generator/feature_builder.cpp | 2 ++ generator/osm2type.cpp | 10 ++++-- generator/search_index_builder.cpp | 51 +++++++++++++++++++----------- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/generator/feature_builder.cpp b/generator/feature_builder.cpp index 0866980a10..b42a5ff954 100644 --- a/generator/feature_builder.cpp +++ b/generator/feature_builder.cpp @@ -135,6 +135,8 @@ void FeatureBuilder::AddStreet(string const & streetName) { m_params.AddStreet(s void FeatureBuilder::AddPostcode(string const & postcode) { + m_params.AddPostcode(postcode); + // todo @t.yan: remove when we stop to add postcodes to metadata m_params.GetMetadata().Set(Metadata::FMD_POSTCODE, postcode); } diff --git a/generator/osm2type.cpp b/generator/osm2type.cpp index 49d72595f7..5650148115 100644 --- a/generator/osm2type.cpp +++ b/generator/osm2type.cpp @@ -867,9 +867,13 @@ void GetNameAndType(OsmElement * p, FeatureParams & params, function const & fn) +bool InsertPostcodes(FeatureType & f, vector const & addrs, + function const & fn) { using namespace search; auto const & postBoxChecker = ftypes::IsPostBoxChecker::Instance(); - string const postcode = f.GetMetadata().Get(feature::Metadata::FMD_POSTCODE); + string const postcode = addrs[f.GetID().m_index].Get(feature::AddressData::Type::Postcode); vector postcodes; if (!postcode.empty()) postcodes.push_back(postcode); @@ -339,9 +340,11 @@ class FeatureInserter { public: FeatureInserter(SynonymsHolder * synonyms, vector> & keyValuePairs, - CategoriesHolder const & catHolder, pair const & scales) + vector const & addrs, CategoriesHolder const & catHolder, + pair const & scales) : m_synonyms(synonyms) , m_keyValuePairs(keyValuePairs) + , m_addrs(addrs) , m_categories(catHolder) , m_scales(scales) { @@ -371,7 +374,7 @@ public: m_keyValuePairs, hasStreetType); bool const useNameAsPostcode = InsertPostcodes( - f, [&inserter](auto const & token) { inserter.AddToken(kPostcodesLang, token); }); + f, m_addrs, [&inserter](auto const & token) { inserter.AddToken(kPostcodesLang, token); }); if (!useNameAsPostcode) f.ForEachName(inserter); @@ -419,6 +422,8 @@ private: SynonymsHolder * m_synonyms; vector> & m_keyValuePairs; + vector const & m_addrs; + CategoriesHolder const & m_categories; pair m_scales; @@ -426,6 +431,7 @@ private: template void AddFeatureNameIndexPairs(FeaturesVectorTest const & features, + vector const & addrs, CategoriesHolder const & categoriesHolder, vector> & keyValuePairs) { @@ -436,7 +442,17 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest const & features, synonyms.reset(new SynonymsHolder(base::JoinPath(GetPlatform().ResourcesDir(), SYNONYMS_FILE))); features.GetVector().ForEach(FeatureInserter( - synonyms.get(), keyValuePairs, categoriesHolder, header.GetScaleRange())); + synonyms.get(), keyValuePairs, addrs, categoriesHolder, header.GetScaleRange())); +} + +void ReadAddressData(FilesContainerR & container, vector & addrs) +{ + ReaderSource src(container.GetReader(SEARCH_TOKENS_FILE_TAG)); + while (src.Size() > 0) + { + addrs.push_back({}); + addrs.back().Deserialize(src); + } } bool GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, string const & streetName, @@ -466,16 +482,9 @@ bool GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, string const & return false; } -void BuildAddressTable(FilesContainerR & container, Writer & writer, uint32_t threadsCount) +void BuildAddressTable(FilesContainerR & container, vector const & addrs, + Writer & writer, uint32_t threadsCount) { - // Read all street names to memory. - ReaderSource src(container.GetReader(SEARCH_TOKENS_FILE_TAG)); - vector addrs; - while (src.Size() > 0) - { - addrs.push_back({}); - addrs.back().Deserialize(src); - } uint32_t const featuresCount = base::checked_cast(addrs.size()); // Initialize temporary source for the current mwm file. @@ -580,7 +589,8 @@ void BuildAddressTable(FilesContainerR & container, Writer & writer, uint32_t th namespace indexer { -void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter); +void BuildSearchIndex(FilesContainerR & container, vector const & addrs, + Writer & indexWriter); bool BuildPostcodesImpl(FilesContainerR & container, storage::CountryId const & country, string const & dataset, string const & tmpFileName, storage::CountryInfoGetter & infoGetter, Writer & indexWriter); @@ -600,15 +610,17 @@ bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild, ui try { + vector addrs; + ReadAddressData(readContainer, addrs); { FileWriter writer(indexFilePath); - BuildSearchIndex(readContainer, writer); + BuildSearchIndex(readContainer, addrs, writer); LOG(LINFO, ("Search index size =", writer.Size())); } if (filename != WORLD_FILE_NAME && filename != WORLD_COASTS_FILE_NAME) { FileWriter writer(addrFilePath); - BuildAddressTable(readContainer, writer, threadsCount); + BuildAddressTable(readContainer, addrs, writer, threadsCount); LOG(LINFO, ("Search address table size =", writer.Size())); } { @@ -770,7 +782,8 @@ bool BuildPostcodesImpl(FilesContainerR & container, storage::CountryId const & return true; } -void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter) +void BuildSearchIndex(FilesContainerR & container, vector const & addrs, + Writer & indexWriter) { using Key = strings::UniString; using Value = Uint64IndexValue; @@ -784,7 +797,7 @@ void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter) SingleValueSerializer serializer; vector> searchIndexKeyValuePairs; - AddFeatureNameIndexPairs(features, categoriesHolder, searchIndexKeyValuePairs); + AddFeatureNameIndexPairs(features, addrs, categoriesHolder, searchIndexKeyValuePairs); sort(searchIndexKeyValuePairs.begin(), searchIndexKeyValuePairs.end()); LOG(LINFO, ("End sorting strings:", timer.ElapsedSeconds()));