From 1632ef8b0d735ccf419ce3c020392a0cc19cbe54 Mon Sep 17 00:00:00 2001 From: Anatoly Serdtcev Date: Thu, 25 Apr 2019 12:32:59 +0300 Subject: [PATCH] [generator:geo_object] Filtrate addressless buildings --- 3party/jansson/myjansson.cpp | 7 +++ 3party/jansson/myjansson.hpp | 1 + generator/generator_tool/generator_tool.cpp | 5 +- generator/geo_objects/geo_objects.cpp | 52 +++++++++++++++++++-- generator/geo_objects/geo_objects.hpp | 6 ++- 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/3party/jansson/myjansson.cpp b/3party/jansson/myjansson.cpp index e6a2ebb4cb..29440aa15c 100644 --- a/3party/jansson/myjansson.cpp +++ b/3party/jansson/myjansson.cpp @@ -78,6 +78,13 @@ string FromJSONToString(json_t * root) namespace std { +void FromJSON(json_t * root, std::string_view & result) +{ + if (!json_is_string(root)) + MYTHROW(base::Json::Exception, ("The field must contain a json string.")); + result = json_string_value(root); +} + void FromJSON(json_t * root, string & result) { if (!json_is_string(root)) diff --git a/3party/jansson/myjansson.hpp b/3party/jansson/myjansson.hpp index 6f580671b0..1f12f5e70a 100644 --- a/3party/jansson/myjansson.hpp +++ b/3party/jansson/myjansson.hpp @@ -272,6 +272,7 @@ struct JSONFreeDeleter namespace std { +void FromJSON(json_t * root, std::string_view & result); void FromJSON(json_t * root, std::string & result); inline base::JSONPtr ToJSON(std::string const & s) { return base::NewJSONString(s); } } // namespace std diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index ac0e1294b6..ffdec1a7dc 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -199,6 +199,8 @@ DEFINE_string(regions_key_value, "", "Input regions key-value file."); DEFINE_string(geo_objects_features, "", "Input tmp.mwm file with geo objects."); DEFINE_string(ids_without_addresses, "", "Output file with objects ids without addresses."); DEFINE_string(geo_objects_key_value, "", "Output geo objects key-value file."); +DEFINE_string(allow_addressless_for_countries, "*", + "Allow addressless buildings for only specified countries separated by commas."); DEFINE_string(regions_features, "", "Input tmp.mwm file with regions."); @@ -390,7 +392,8 @@ int GeneratorToolMain(int argc, char ** argv) { if (!geo_objects::GenerateGeoObjects(FLAGS_regions_index, FLAGS_regions_key_value, FLAGS_geo_objects_features, FLAGS_ids_without_addresses, - FLAGS_geo_objects_key_value, FLAGS_verbose)) + FLAGS_geo_objects_key_value, + FLAGS_allow_addressless_for_countries, FLAGS_verbose)) return EXIT_FAILURE; } diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp index 643cd4313b..4a085d83d6 100644 --- a/generator/geo_objects/geo_objects.cpp +++ b/generator/geo_objects/geo_objects.cpp @@ -1,6 +1,7 @@ #include "generator/geo_objects/geo_objects.hpp" #include "generator/feature_builder.hpp" +#include "generator/feature_generator.hpp" #include "generator/geo_objects/geo_object_info_getter.hpp" #include "generator/geo_objects/geo_objects_filter.hpp" #include "generator/geo_objects/key_value_storage.hpp" @@ -133,6 +134,40 @@ MakeTempGeoObjectsIndex(std::string const & pathToGeoObjectsTmpMwm) return indexer::ReadIndex, MmapReader>(indexFile); } +void FilterAddresslessByCountryAndRepackMwm(std::string const & pathInGeoObjectsTmpMwm, + std::string_view const & includeCountries, + RegionInfoGetter const & regionInfoGetter) +{ + auto const path = Platform().TmpPathForFile(); + feature::FeaturesCollector collector(path); + + auto const filteringCollector = [&](FeatureBuilder1 const & fb, uint64_t /* currPos */) + { + if (GeoObjectsFilter::HasHouse(fb)) + { + collector(fb); + return; + } + + auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint()); + if (!regionKeyValue) + return; + + auto && properties = base::GetJSONObligatoryField(regionKeyValue->second.get(), "properties"); + auto && address = base::GetJSONObligatoryField(properties, "address"); + auto && country = base::GetJSONObligatoryField(address, "country"); + auto countryName = FromJSON(country); + auto pos = includeCountries.find(countryName); + if (pos != std::string_view::npos) + collector(fb); + }; + feature::ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, filteringCollector); + + Platform().RemoveFileIfExists(pathInGeoObjectsTmpMwm); + if (std::rename(path.c_str(), pathInGeoObjectsTmpMwm.c_str()) != 0) + LOG(LERROR, ("Error: Cannot rename", path, "to", pathInGeoObjectsTmpMwm)); +} + void BuildGeoObjectsWithAddresses(RegionInfoGetter const & regionInfoGetter, std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamGeoObjectsKv, bool) @@ -196,7 +231,8 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv, std::string const & pathInGeoObjectsTmpMwm, std::string const & pathOutIdsWithoutAddress, - std::string const & pathOutGeoObjectsKv, bool verbose) + std::string const & pathOutGeoObjectsKv, + std::string const & allowAddresslessForCountries, bool verbose) { LOG(LINFO, ("Start generating geo objects..")); auto timer = base::Timer(); @@ -204,12 +240,20 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex, LOG(LINFO, ("Finish generating geo objects.", timer.ElapsedSeconds(), "seconds.")); }); - auto geoObjectIndexFuture = std::async(std::launch::async, MakeTempGeoObjectsIndex, - pathInGeoObjectsTmpMwm); - RegionInfoGetter regionInfoGetter{pathInRegionsIndex, pathInRegionsKv}; LOG(LINFO, ("Size of regions key-value storage:", regionInfoGetter.GetStorage().Size())); + if (allowAddresslessForCountries != "*") + { + FilterAddresslessByCountryAndRepackMwm(pathInGeoObjectsTmpMwm, allowAddresslessForCountries, + regionInfoGetter); + LOG(LINFO, ("Addressless buildings are filtered except countries", + allowAddresslessForCountries, ".")); + } + + auto geoObjectIndexFuture = std::async(std::launch::async, MakeTempGeoObjectsIndex, + pathInGeoObjectsTmpMwm); + std::ofstream streamGeoObjectsKv(pathOutGeoObjectsKv); BuildStreets(regionInfoGetter, pathInGeoObjectsTmpMwm, streamGeoObjectsKv, verbose); LOG(LINFO, ("Streets were built.")); diff --git a/generator/geo_objects/geo_objects.hpp b/generator/geo_objects/geo_objects.hpp index 1b3922cc4d..90e779eb42 100644 --- a/generator/geo_objects/geo_objects.hpp +++ b/generator/geo_objects/geo_objects.hpp @@ -11,10 +11,14 @@ namespace geo_objects // In this step, we need key-value pairs for the regions and the index for the regions. // Then we build an index for houses. And then we finish building key-value pairs for poi using // this index for houses. +// |allowAddresslessForCountries| specifies countries for which addressless buldings are constructed +// in index and key-value files. Countries are specified by osm's default local name (or part of name) +// separated by commas. Default value is '*' (for all countries). bool GenerateGeoObjects(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv, std::string const & pathInGeoObjectsTmpMwm, std::string const & pathOutIdsWithoutAddress, - std::string const & pathOutGeoObjectsKv, bool verbose); + std::string const & pathOutGeoObjectsKv, + std::string const & allowAddresslessForCountries, bool verbose); } // namespace geo_objects } // namespace generator