From 8f52a34c01b5952011b9013181114e85190cd008 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Thu, 18 Oct 2018 13:43:50 +0300 Subject: [PATCH] [generator] Added repacking regions mwm.tmp. --- generator/generator_tool/generator_tool.cpp | 29 ++++++----- generator/regions/regions.cpp | 53 ++++++++++++++------- generator/regions/regions.hpp | 10 ++-- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 5f93a33d7f..576739cfe2 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -16,6 +16,7 @@ #include "generator/metalines_builder.hpp" #include "generator/osm_source.hpp" #include "generator/popular_places_section_builder.hpp" +#include "generator/regions/collector_region_info.hpp" #include "generator/regions/regions.hpp" #include "generator/restriction_generator.hpp" #include "generator/road_access_generator.hpp" @@ -388,6 +389,23 @@ int main(int argc, char ** argv) } } + if (FLAGS_generate_regions_kv) + { + CHECK(FLAGS_generate_region_features, ("Option --generate_regions_kv can be used only " + "together with option --generate_region_features.")); + auto const pathInRegionsCollector = genInfo.GetTmpFileName(genInfo.m_fileName, + regions::CollectorRegionInfo::kDefaultExt); + auto const pathInRegionsTmpMwm = genInfo.GetTmpFileName(genInfo.m_fileName); + auto const pathOutRepackedRegionsTmpMwm = genInfo.GetTmpFileName(genInfo.m_fileName + "_repacked"); + auto const pathOutRegionsKv = genInfo.GetIntermediateFileName(genInfo.m_fileName, ".jsonl"); + if (!regions::GenerateRegions(pathInRegionsTmpMwm, pathInRegionsCollector, pathOutRegionsKv, + pathOutRepackedRegionsTmpMwm, FLAGS_verbose)) + { + LOG(LCRITICAL, ("Error generating regions kv.")); + return EXIT_FAILURE; + } + } + // Enumerate over all dat files that were created. size_t const count = genInfo.m_bucketNames.size(); for (size_t i = 0; i < count; ++i) @@ -426,17 +444,6 @@ int main(int argc, char ** argv) } } - if (FLAGS_generate_regions_kv) - { - CHECK(FLAGS_generate_region_features, ("Option --generate_regions_kv can be used only " - "together with option --generate_region_features.")); - if (!regions::GenerateRegions(genInfo)) - { - LOG(LCRITICAL, ("Error generating regions kv.")); - return EXIT_FAILURE; - } - } - if (FLAGS_generate_index) { LOG(LINFO, ("Generating index for", datFile)); diff --git a/generator/regions/regions.cpp b/generator/regions/regions.cpp index 85135476f3..b642b91653 100644 --- a/generator/regions/regions.cpp +++ b/generator/regions/regions.cpp @@ -1,6 +1,7 @@ #include "generator/regions/regions.hpp" #include "generator/feature_builder.hpp" +#include "generator/feature_generator.hpp" #include "generator/generate_info.hpp" #include "generator/regions/city.hpp" #include "generator/regions/node.hpp" @@ -22,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -134,11 +134,10 @@ private: }; std::tuple -ReadDatasetFromTmpMwm(feature::GenerateInfo const & genInfo, RegionInfo & collector) +ReadDatasetFromTmpMwm(std::string const & tmpMwmFilename, RegionInfo & collector) { RegionsBuilder::Regions regions; PointCitiesMap pointCitiesMap; - auto const tmpMwmFilename = genInfo.GetTmpFileName(genInfo.m_fileName); auto const toDo = [®ions, &pointCitiesMap, &collector](FeatureBuilder1 const & fb, uint64_t /* currPos */) { if (fb.IsArea() && fb.IsGeometryClosed()) @@ -170,37 +169,50 @@ void FilterRegions(RegionsBuilder::Regions & regions) regions.erase(it, std::end(regions)); } -RegionsBuilder::Regions ReadData(feature::GenerateInfo const & genInfo, +RegionsBuilder::Regions ReadData(std::string const & tmpMwmFilename, RegionInfo & regionsInfoCollector) { RegionsBuilder::Regions regions; PointCitiesMap pointCitiesMap; - std::tie(regions, pointCitiesMap) = ReadDatasetFromTmpMwm(genInfo, regionsInfoCollector); + std::tie(regions, pointCitiesMap) = ReadDatasetFromTmpMwm(tmpMwmFilename, regionsInfoCollector); RegionsFixer fixer(regions, pointCitiesMap); regions = fixer.FixRegions(); FilterRegions(regions); return regions; } + +void RepackTmpMwm(std::string const & srcFilename, std::string const & repackedFilename, + std::set const & ids) +{ + feature::FeaturesCollector collector(repackedFilename); + auto const toDo = [&collector, &ids](FeatureBuilder1 const & fb, uint64_t /* currPos */) + { + if (ids.find(fb.GetMostGenericOsmId()) != std::end(ids)) + collector(fb); + }; + + feature::ForEachFromDatRawFormat(srcFilename, toDo); +} } // namespace -bool GenerateRegions(feature::GenerateInfo const & genInfo) +bool GenerateRegions(std::string const & pathInRegionsTmpMwm, + std::string const & pathInRegionsCollector, + std::string const & pathOutRegionsKv, + std::string const & pathOutRepackedRegionsTmpMwm, bool verbose) { using namespace regions; - LOG(LINFO, ("Start generating regions..")); + LOG(LINFO, ("Start generating regions from ", pathInRegionsTmpMwm)); auto timer = base::Timer(); - Transliteration::Instance().Init(GetPlatform().ResourcesDir()); - auto const collectorFilename = genInfo.GetTmpFileName(genInfo.m_fileName, - CollectorRegionInfo::kDefaultExt); - RegionInfo regionsInfoCollector(collectorFilename); - RegionsBuilder::Regions regions = ReadData(genInfo, regionsInfoCollector); - auto jsonPolicy = std::make_unique(genInfo.m_verbose); + + RegionInfo regionsInfoCollector(pathInRegionsCollector); + RegionsBuilder::Regions regions = ReadData(pathInRegionsTmpMwm, regionsInfoCollector); + auto jsonPolicy = std::make_unique(verbose); auto kvBuilder = std::make_unique(std::move(regions), std::move(jsonPolicy)); auto const countryTrees = kvBuilder->GetCountryTrees(); - auto const jsonlName = genInfo.GetIntermediateFileName(genInfo.m_fileName, ".jsonl"); - std::ofstream ofs(jsonlName, std::ofstream::out); + std::ofstream ofs(pathOutRegionsKv, std::ofstream::out); std::set setIds; size_t countIds = 0; for (auto const & countryName : kvBuilder->GetCountryNames()) @@ -209,7 +221,7 @@ bool GenerateRegions(feature::GenerateInfo const & genInfo) if (!tree) continue; - if (genInfo.m_verbose) + if (verbose) DebugPrintTree(tree); auto const idStringList = kvBuilder->ToIdStringList(tree); @@ -222,6 +234,15 @@ bool GenerateRegions(feature::GenerateInfo const & genInfo) } } + // todo(maksimandrianov1): Perhaps this is not the best solution. This is a hot fix. Perhaps it + // is better to transfer this to index generation(function GenerateRegionsData), + // or to combine index generation and key-value storage generation in + // generator_tool(generator_tool.cpp). + if (!pathOutRepackedRegionsTmpMwm.empty()) + RepackTmpMwm(pathInRegionsTmpMwm, pathOutRepackedRegionsTmpMwm, setIds); + + LOG(LINFO, ("Regions objects key-value storage saved to", pathOutRegionsKv)); + LOG(LINFO, ("Repacked regions temprory mwm saved to", pathOutRepackedRegionsTmpMwm)); LOG(LINFO, (countIds, "total ids.", setIds.size(), "unique ids.")); LOG(LINFO, ("Finish generating regions.", timer.ElapsedSeconds(), "seconds.")); return true; diff --git a/generator/regions/regions.hpp b/generator/regions/regions.hpp index c87a9cc669..d40b4b189e 100644 --- a/generator/regions/regions.hpp +++ b/generator/regions/regions.hpp @@ -1,14 +1,14 @@ #pragma once -namespace feature -{ -struct GenerateInfo; -} // namespace feature +#include namespace generator { namespace regions { -bool GenerateRegions(feature::GenerateInfo const & genInfo); +bool GenerateRegions(std::string const & pathInRegionsTmpMwm, + std::string const & pathInRegionsCollector, + std::string const & pathOutRegionsKv, + std::string const & pathOutRepackedRegionsTmpMwm, bool verbose); } // namespace regions } // namespace generator