From 5c95543a86fb92a941d76d85cf59fe609ddb1b0d Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Wed, 18 Dec 2019 13:52:19 +0300 Subject: [PATCH] [maps_generator] Separate parameters for us and uk postcodes datasets. --- generator/postcode_points_builder.cpp | 14 +++++++++----- tools/python/maps_generator/generator/env.py | 8 ++++++-- tools/python/maps_generator/generator/gen_tool.py | 3 ++- tools/python/maps_generator/generator/settings.py | 6 ++++-- tools/python/maps_generator/maps_generator.py | 6 ++++-- .../var/etc/map_generator.ini.default | 3 ++- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/generator/postcode_points_builder.cpp b/generator/postcode_points_builder.cpp index 222c0af2fb..afb1a8c22e 100644 --- a/generator/postcode_points_builder.cpp +++ b/generator/postcode_points_builder.cpp @@ -65,10 +65,10 @@ void GetUKPostcodes(string const & filename, storage::CountryId const & countryI CHECK_EQUAL(fields.size(), kDatasetCount, (line)); double lat; - CHECK(strings::to_double(fields[kLatIndex], lat), ()); + CHECK(strings::to_double(fields[kLatIndex], lat), (line)); double lon; - CHECK(strings::to_double(fields[kLongIndex], lon), ()); + CHECK(strings::to_double(fields[kLongIndex], lon), (line)); auto const p = mercator::FromLatLon(lat, lon); @@ -112,7 +112,11 @@ void GetUSPostcodes(string const & filename, storage::CountryId const & countryI data.open(filename); data.exceptions(fstream::badbit); + // Skip header. string line; + if (!getline(data, line)) + return; + size_t index = 0; while (getline(data, line)) { @@ -121,10 +125,10 @@ void GetUSPostcodes(string const & filename, storage::CountryId const & countryI CHECK_EQUAL(fields.size(), kDatasetCount, (line)); double lat; - CHECK(strings::to_double(fields[kLatIndex], lat), ()); + CHECK(strings::to_double(fields[kLatIndex], lat), (line)); double lon; - CHECK(strings::to_double(fields[kLongIndex], lon), ()); + CHECK(strings::to_double(fields[kLongIndex], lon), (line)); auto const p = mercator::FromLatLon(lat, lon); @@ -169,7 +173,7 @@ bool BuildPostcodePointsImpl(FilesContainerR & container, storage::CountryId con vector> usPostcodesKeyValuePairs; vector usPostcodesValueMapping; if (!usDatasetPath.empty()) - GetUKPostcodes(usDatasetPath, country, infoGetter, usPostcodesValueMapping, + GetUSPostcodes(usDatasetPath, country, infoGetter, usPostcodesValueMapping, usPostcodesKeyValuePairs); if (ukPostcodesKeyValuePairs.empty() && usPostcodesKeyValuePairs.empty()) diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 0b16394621..1241b87ccb 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -225,8 +225,12 @@ class Env: return os.path.join(self.intermediate_path, "translations_food.json") @property - def postcodes_path(self): - return os.path.join(self.intermediate_path, "postcodes") + def uk_postcodes_path(self): + return os.path.join(self.intermediate_path, "uk_postcodes") + + @property + def us_postcodes_path(self): + return os.path.join(self.intermediate_path, "us_postcodes") @property def cities_boundaries_path(self): diff --git a/tools/python/maps_generator/generator/gen_tool.py b/tools/python/maps_generator/generator/gen_tool.py index 901442da6b..2cda09f178 100644 --- a/tools/python/maps_generator/generator/gen_tool.py +++ b/tools/python/maps_generator/generator/gen_tool.py @@ -62,13 +62,14 @@ class GenTool: "osm_file_type": str, "output": str, "popular_places_data": str, - "postcodes_dataset": str, "regions_features": str, "regions_index": str, "regions_key_value": str, "srtm_path": str, "transit_path": str, "ugc_data": str, + "uk_postcodes_dataset": str, + "us_postcodes_dataset": str, "user_resource_path": str, "wikipedia_pages": str, } diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index a6f216502b..96fc8dde0b 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -51,7 +51,8 @@ POPULARITY_URL= "" SUBWAY_URL = "" FOOD_URL = "" FOOD_TRANSLATIONS_URL = "" -POSTCODES_URL = "" +UK_POSTCODES_URL = "" +US_POSTCODES_URL = "" SRTM_PATH = "" STATS_TYPES_CONFIG = "" @@ -125,7 +126,8 @@ PROMO_CATALOG_COUNTRIES_URL = _get_opt_path(config, "External", "PROMO_CATALOG_C POPULARITY_URL = _get_opt_path(config, "External", "POPULARITY_URL", POPULARITY_URL) SUBWAY_URL = _get_opt(config, "External", "SUBWAY_URL", SUBWAY_URL) FOOD_URL = _get_opt(config, "External", "FOOD_URL", FOOD_URL) -POSTCODES_URL = _get_opt(config, "External", "POSTCODES_URL", POSTCODES_URL) +UK_POSTCODES_URL = _get_opt(config, "External", "UK_POSTCODES_URL", UK_POSTCODES_URL) +US_POSTCODES_URL = _get_opt(config, "External", "US_POSTCODES_URL", US_POSTCODES_URL) FOOD_TRANSLATIONS_URL = _get_opt(config, "External", "FOOD_TRANSLATIONS_URL", FOOD_TRANSLATIONS_URL) SRTM_PATH = _get_opt_path(config, "External", "SRTM_PATH", SRTM_PATH) diff --git a/tools/python/maps_generator/maps_generator.py b/tools/python/maps_generator/maps_generator.py index 97fd903a5c..a4d7fa1526 100644 --- a/tools/python/maps_generator/maps_generator.py +++ b/tools/python/maps_generator/maps_generator.py @@ -67,7 +67,8 @@ def stage_download_production_external(env): settings.POPULARITY_URL: env.popularity_path, settings.FOOD_URL: env.food_paths, settings.FOOD_TRANSLATIONS_URL: env.food_translations_path, - settings.POSTCODES_URL: env.postcodes_path, + settings.UK_POSTCODES_URL: env.uk_postcodes_path, + settings.US_POSTCODES_URL: env.us_postcodes_path, }) @@ -124,7 +125,8 @@ def stage_index(env, country, **kwargs): else: extra = {} if env.is_accepted_stage(stage_download_production_external): - extra["postcodes_dataset"] = env.postcodes_path + extra["uk_postcodes_dataset"] = env.uk_postcodes_path + extra["us_postcodes_dataset"] = env.us_postcodes_path stages.stage_index(env, country, **kwargs, **extra) diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 8d7bc3f1cd..df15d0149a 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -35,7 +35,8 @@ SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json # POPULARITY_URL: # FOOD_URL: # FOOD_TRANSLATIONS_URL: -# POSTCODES_URL: +# UK_POSTCODES_URL: +# US_POSTCODES_URL: # SRTM_PATH: [Stats]