From 90427d18137c7db18134457a17b91467450b7afe Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Tue, 29 Dec 2020 12:47:46 +0300 Subject: [PATCH 01/28] Uncommented tests. --- .../features_tests.cpp | 64 +++++++++---------- .../routing_benchmarks/car_routing_tests.cpp | 20 +++--- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/generator/generator_integration_tests/features_tests.cpp b/generator/generator_integration_tests/features_tests.cpp index c6e1051239..89b8c54f08 100644 --- a/generator/generator_integration_tests/features_tests.cpp +++ b/generator/generator_integration_tests/features_tests.cpp @@ -519,47 +519,47 @@ private: feature::GenerateInfo m_genInfo; }; -//UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCoasts) -//{ -// FeatureIntegrationTests::BuildCoasts(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCoasts) +{ + FeatureIntegrationTests::BuildCoasts(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, BuildWorldMultithread) -//{ -// FeatureIntegrationTests::BuildWorld(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, BuildWorldMultithread) +{ + FeatureIntegrationTests::BuildWorld(); +} UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCountriesMultithread) { FeatureIntegrationTests::BuildCountries(); } -//UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCountriesWithComplex) -//{ -// FeatureIntegrationTests::BuildCountriesWithComplex(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCountriesWithComplex) +{ + FeatureIntegrationTests::BuildCountriesWithComplex(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, CheckMixedTagsAndNodes) -//{ -// FeatureIntegrationTests::CheckMixedTagsAndNodes(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, CheckMixedTagsAndNodes) +{ + FeatureIntegrationTests::CheckMixedTagsAndNodes(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, CheckGeneratedDataMultithread) -//{ -// FeatureIntegrationTests::CheckGeneratedData(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, CheckGeneratedDataMultithread) +{ + FeatureIntegrationTests::CheckGeneratedData(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, BuildWorldOneThread) -//{ -// FeatureIntegrationTests::BuildWorldOneThread(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, BuildWorldOneThread) +{ + FeatureIntegrationTests::BuildWorldOneThread(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCountriesOneThread) -//{ -// FeatureIntegrationTests::BuildCountriesOneThread(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, BuildCountriesOneThread) +{ + FeatureIntegrationTests::BuildCountriesOneThread(); +} -//UNIT_CLASS_TEST(FeatureIntegrationTests, CheckGeneratedDataOneThread) -//{ -// FeatureIntegrationTests::CheckGeneratedDataOneThread(); -//} +UNIT_CLASS_TEST(FeatureIntegrationTests, CheckGeneratedDataOneThread) +{ + FeatureIntegrationTests::CheckGeneratedDataOneThread(); +} diff --git a/routing/routing_benchmarks/car_routing_tests.cpp b/routing/routing_benchmarks/car_routing_tests.cpp index 297ea28bf8..ec8cd4546b 100644 --- a/routing/routing_benchmarks/car_routing_tests.cpp +++ b/routing/routing_benchmarks/car_routing_tests.cpp @@ -60,15 +60,15 @@ UNIT_CLASS_TEST(CarTest, InCity) TestCarRouter(ms::LatLon(55.75785, 37.58267), ms::LatLon(55.76082, 37.58492), 30); } -//// Start and finish are located near a big road. -//UNIT_CLASS_TEST(CarTest, BigRoad) -//{ -// TestCarRouter(ms::LatLon(55.75826, 37.39476), ms::LatLon(55.7605, 37.39003), 30); -//} +// Start and finish are located near a big road. +UNIT_CLASS_TEST(CarTest, BigRoad) +{ + TestCarRouter(ms::LatLon(55.75826, 37.39476), ms::LatLon(55.7605, 37.39003), 30); +} -//// Start are located near an airport center. It's far from road network. -//UNIT_CLASS_TEST(CarTest, InAirport) -//{ -// TestCarRouter(ms::LatLon(55.97285, 37.41275), ms::LatLon(55.96396, 37.41922), 30); -//} +// Start are located near an airport center. It's far from road network. +UNIT_CLASS_TEST(CarTest, InAirport) +{ + TestCarRouter(ms::LatLon(55.97285, 37.41275), ms::LatLon(55.96396, 37.41922), 30); +} } // namespace From 319864f0b72a38ef8f5efb396b62231b569aac09 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:00:18 +0300 Subject: [PATCH 02/28] [generator] BUILD_WORLD_ROADS param in ini. --- .../python/maps_generator/var/etc/map_generator.ini.default | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index fd3c447e07..7bb36bacf1 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -57,15 +57,15 @@ DIFF_VERSION_DEPTH: 2 # The https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and # the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom. # PLANET_COASTS_URL: +# Set to 'true' to build special routing section in World.mwm for alerting about absent regions without which the +# route can't be built. +BUILD_WORLD_ROADS: true # The url to the subway file. SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json # The url of the location with the transit files extracted from GTFS. # TRANSIT_URL: -# The url of the file with info about main roads between mwms for special routing section in World mwm. -# WORLDROADS_URL: - # Urls for production maps generation. # UGC_URL: # HOTELS_URL: From a96976c240e190cd2bb006d00db9ec133c9e5d71 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:00:47 +0300 Subject: [PATCH 03/28] [generator_tool] Renamed world_roads_path param. --- generator/generator_tool/generator_tool.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 75730a6795..b71eed20b2 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -151,7 +151,7 @@ DEFINE_bool(disable_cross_mwm_progress, false, DEFINE_string(srtm_path, "", "Path to srtm directory. If set, generates a section with altitude information " "about roads."); -DEFINE_string(worldroads_path, "", +DEFINE_string(world_roads_path, "", "Path to a file with roads that should end up on the world map. If set, generates a " "section with these roads in World.mwm. The roads may be used to identify which mwm " "files are touched by an arbitrary route."); @@ -491,10 +491,10 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) } } - if (country == WORLD_FILE_NAME && !FLAGS_worldroads_path.empty()) + if (country == WORLD_FILE_NAME && !FLAGS_world_roads_path.empty()) { LOG(LINFO, ("Generating routing section for World.")); - if (!routing::BuildWorldRoads(dataFile, FLAGS_worldroads_path)) + if (!routing::BuildWorldRoads(dataFile, FLAGS_world_roads_path)) { LOG(LCRITICAL, ("Generating routing section for World has failed.")); return EXIT_FAILURE; From 75b96f55ff9c1729e07f358e610371ad06ce6ea8 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:02:10 +0300 Subject: [PATCH 04/28] [generator] Replace WORLDROADS_URL with BUILD_WORLD_ROADS in settings. --- tools/python/maps_generator/generator/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index 0b7a38f5cb..09e2e50f02 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -115,7 +115,7 @@ PROMO_CATALOG_COUNTRIES_URL = "" POPULARITY_URL = "" SUBWAY_URL = "" TRANSIT_URL = "" -WORLDROADS_URL = "" +BUILD_WORLD_ROADS = True FOOD_URL = "" FOOD_TRANSLATIONS_URL = "" UK_POSTCODES_URL = "" @@ -277,7 +277,7 @@ def init(default_settings_path: AnyStr): global POPULARITY_URL global SUBWAY_URL global TRANSIT_URL - global WORLDROADS_URL + global BUILD_WORLD_ROADS global FOOD_URL global UK_POSTCODES_URL global US_POSTCODES_URL @@ -301,7 +301,7 @@ def init(default_settings_path: AnyStr): POPULARITY_URL = cfg.get_opt_path("External", "POPULARITY_URL", POPULARITY_URL) SUBWAY_URL = cfg.get_opt("External", "SUBWAY_URL", SUBWAY_URL) TRANSIT_URL = cfg.get_opt("External", "TRANSIT_URL", TRANSIT_URL) - WORLDROADS_URL = cfg.get_opt("External", "WORLDROADS_URL", WORLDROADS_URL) + BUILD_WORLD_ROADS = cfg.get_opt("External", "BUILD_WORLD_ROADS", BUILD_WORLD_ROADS) FOOD_URL = cfg.get_opt("External", "FOOD_URL", FOOD_URL) UK_POSTCODES_URL = cfg.get_opt("External", "UK_POSTCODES_URL", UK_POSTCODES_URL) From c79da9c04a20e40a36078a90a62f7a2ef1895426 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:03:21 +0300 Subject: [PATCH 05/28] [generator] Rename world_roads_path. --- tools/python/maps_generator/generator/gen_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/python/maps_generator/generator/gen_tool.py b/tools/python/maps_generator/generator/gen_tool.py index 3f4de37c43..b11216e3b1 100644 --- a/tools/python/maps_generator/generator/gen_tool.py +++ b/tools/python/maps_generator/generator/gen_tool.py @@ -74,7 +74,7 @@ class GenTool: "srtm_path": str, "transit_path": str, "transit_path_experimental": str, - "worldroads_path": str, + "world_roads_path": str, "ugc_data": str, "uk_postcodes_dataset": str, "us_postcodes_dataset": str, From 94a6233ce5303fff239104454a436499648a98f6 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:04:55 +0300 Subject: [PATCH 06/28] [generator] New step step_prepare_routing_world. --- .../python/maps_generator/generator/steps.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 94567b4484..47e4c9a9b2 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -200,6 +200,23 @@ def step_cities_ids_world(env: Env, country: AnyStr, **kwargs): ) +def step_prepare_routing_world(env: Env, country: AnyStr, **kwargs): + world_roads_builder_tool_with_args = [env.world_roads_builder_tool, + f"--path_roads_file={env.paths.planet_o5m}", + f"--path_resources={env.paths.user_resource_path}", + f"--path_res_file={env.paths.world_roads_path}"] + logger.info(f"Starting {world_roads_builder_tool_with_args}") + sub_proc = subprocess.Popen( + world_roads_builder_tool_with_args, + stdout=env.get_subprocess_out(country), + stderr=env.get_subprocess_out(country), + env=os.environ + ) + + if sub_proc.wait() != 0: + raise Exception(f"Error running {world_roads_builder_tool_with_args}") + + def step_routing_world(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, @@ -209,7 +226,7 @@ def step_routing_world(env: Env, country: AnyStr, **kwargs): data_path=env.paths.mwm_path, user_resource_path=env.paths.user_resource_path, output=country, - worldroads_path=env.paths.worldroads_path, + world_roads_path=env.paths.world_roads_path, **kwargs, ) From be86938291529181de0a44bbe1edcb4c3fb61f6a Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:05:37 +0300 Subject: [PATCH 07/28] [generator] Changed stages declaration. --- .../maps_generator/generator/stages_declaration.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index 1468abf2c2..b2fe029441 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -179,7 +179,8 @@ class StageMwm(Stage): @staticmethod def make_mwm(country: AnyStr, env: Env): world_stages = { - WORLD_NAME: [StageIndex, StageCitiesIdsWorld, StageRoutingWorld, StageMwmStatistics], + WORLD_NAME: [StageIndex, StageCitiesIdsWorld, StagePrepareRoutingWorld, StageRoutingWorld, + StageMwmStatistics], WORLD_COASTS_NAME: [StageIndex, StageMwmStatistics], } @@ -232,10 +233,17 @@ class StageCitiesIdsWorld(Stage): @country_stage -@depends_from_internal(D(settings.WORLDROADS_URL, PathProvider.worldroads_path),) +class StagePrepareRoutingWorld(Stage): + def apply(self, env: Env, country, **kwargs): + if env.paths.need_to_build_world_roads: + steps.step_prepare_routing_world(env, country, **kwargs) + + +@country_stage class StageRoutingWorld(Stage): def apply(self, env: Env, country, **kwargs): - steps.step_routing_world(env, country, **kwargs) + if env.paths.need_to_build_world_roads: + steps.step_routing_world(env, country, **kwargs) @country_stage From 6add0b136d523659a3d4608367e00740f3d9d33e Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 29 Dec 2020 12:06:00 +0300 Subject: [PATCH 08/28] [generator] setup_world_roads_builder_tool. --- tools/python/maps_generator/generator/env.py | 21 +++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 52808d7024..775cf4aef7 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -231,13 +231,17 @@ class PathProvider: ) @property - def worldroads_path(self) -> AnyStr: + def world_roads_path(self) -> AnyStr: return ( - os.path.join(self.intermediate_data_path, "worldroads.txt") - if settings.WORLDROADS_URL + os.path.join(self.intermediate_data_path, "world_roads.txt") + if settings.BUILD_WORLD_ROADS else "" ) + @property + def need_to_build_world_roads(self) -> bool: + return settings.BUILD_WORLD_ROADS + @property def planet_osm_pbf(self) -> AnyStr: return os.path.join(self.build_path, f"{settings.PLANET}.osm.pbf") @@ -434,6 +438,9 @@ class Env: if item.endswith(".download"): os.remove(os.path.join(self.paths.status_path, item)) + if self.paths.need_to_build_world_roads: + self.world_roads_builder_tool = self.setup_world_roads_builder_tool() + self.main_status = status.Status() # self.countries_meta stores log files and statuses for each country. self.countries_meta = collections.defaultdict(dict) @@ -523,6 +530,14 @@ class Env: raise Exception(exceptions) + @staticmethod + def setup_world_roads_builder_tool() -> AnyStr: + logger.info(f"Check world_roads_builder_tool. Looking for it in {settings.BUILD_PATH} ...") + world_roads_builder_tool_path = find_executable(settings.BUILD_PATH, "world_roads_builder_tool") + logger.info(f"world_roads_builder_tool found - {world_roads_builder_tool_path}") + return world_roads_builder_tool_path + + @staticmethod def setup_osm_tools() -> Dict[AnyStr, AnyStr]: path = settings.OSM_TOOLS_PATH From 1c5297ce869a6ea91fe77eefa4ee642073714f38 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Wed, 30 Dec 2020 17:21:33 +0300 Subject: [PATCH 09/28] [generator] Renamed NEED_BUILD_WORLD_ROADS and refactored stage as helper_stage. --- tools/python/maps_generator/__main__.py | 3 +++ tools/python/maps_generator/generator/env.py | 9 ++------- tools/python/maps_generator/generator/settings.py | 6 +++--- .../maps_generator/generator/stages_declaration.py | 7 +++---- tools/python/maps_generator/generator/steps.py | 5 ++--- .../maps_generator/var/etc/map_generator.ini.default | 2 +- 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tools/python/maps_generator/__main__.py b/tools/python/maps_generator/__main__.py index 0cedd6d164..5734b08d21 100644 --- a/tools/python/maps_generator/__main__.py +++ b/tools/python/maps_generator/__main__.py @@ -278,6 +278,9 @@ def main(): if not settings.NEED_PLANET_UPDATE: skipped_stages.add(sd.StageUpdatePlanet) + if not settings.NEED_BUILD_WORLD_ROADS: + skipped_stages.add(sd.StageRoutingWorld) + # Make env and run maps generation. env = Env( countries=countries, diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 775cf4aef7..cb63c545ed 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -234,14 +234,10 @@ class PathProvider: def world_roads_path(self) -> AnyStr: return ( os.path.join(self.intermediate_data_path, "world_roads.txt") - if settings.BUILD_WORLD_ROADS + if settings.NEED_BUILD_WORLD_ROADS else "" ) - @property - def need_to_build_world_roads(self) -> bool: - return settings.BUILD_WORLD_ROADS - @property def planet_osm_pbf(self) -> AnyStr: return os.path.join(self.build_path, f"{settings.PLANET}.osm.pbf") @@ -438,8 +434,7 @@ class Env: if item.endswith(".download"): os.remove(os.path.join(self.paths.status_path, item)) - if self.paths.need_to_build_world_roads: - self.world_roads_builder_tool = self.setup_world_roads_builder_tool() + self.world_roads_builder_tool = self.setup_world_roads_builder_tool() self.main_status = status.Status() # self.countries_meta stores log files and statuses for each country. diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index 09e2e50f02..a16dd0a0fd 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -115,7 +115,7 @@ PROMO_CATALOG_COUNTRIES_URL = "" POPULARITY_URL = "" SUBWAY_URL = "" TRANSIT_URL = "" -BUILD_WORLD_ROADS = True +NEED_BUILD_WORLD_ROADS = True FOOD_URL = "" FOOD_TRANSLATIONS_URL = "" UK_POSTCODES_URL = "" @@ -277,7 +277,7 @@ def init(default_settings_path: AnyStr): global POPULARITY_URL global SUBWAY_URL global TRANSIT_URL - global BUILD_WORLD_ROADS + global NEED_BUILD_WORLD_ROADS global FOOD_URL global UK_POSTCODES_URL global US_POSTCODES_URL @@ -301,7 +301,7 @@ def init(default_settings_path: AnyStr): POPULARITY_URL = cfg.get_opt_path("External", "POPULARITY_URL", POPULARITY_URL) SUBWAY_URL = cfg.get_opt("External", "SUBWAY_URL", SUBWAY_URL) TRANSIT_URL = cfg.get_opt("External", "TRANSIT_URL", TRANSIT_URL) - BUILD_WORLD_ROADS = cfg.get_opt("External", "BUILD_WORLD_ROADS", BUILD_WORLD_ROADS) + NEED_BUILD_WORLD_ROADS = cfg.get_opt("External", "NEED_BUILD_WORLD_ROADS", NEED_BUILD_WORLD_ROADS) FOOD_URL = cfg.get_opt("External", "FOOD_URL", FOOD_URL) UK_POSTCODES_URL = cfg.get_opt("External", "UK_POSTCODES_URL", UK_POSTCODES_URL) diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index b2fe029441..5fbbba0da3 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -233,17 +233,16 @@ class StageCitiesIdsWorld(Stage): @country_stage +@helper_stage_for("StageRoutingWorld") class StagePrepareRoutingWorld(Stage): def apply(self, env: Env, country, **kwargs): - if env.paths.need_to_build_world_roads: - steps.step_prepare_routing_world(env, country, **kwargs) + steps.step_prepare_routing_world(env, country, **kwargs) @country_stage class StageRoutingWorld(Stage): def apply(self, env: Env, country, **kwargs): - if env.paths.need_to_build_world_roads: - steps.step_routing_world(env, country, **kwargs) + steps.step_routing_world(env, country, **kwargs) @country_stage diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 47e4c9a9b2..52a85917b9 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -15,7 +15,7 @@ from maps_generator.generator.env import PathProvider from maps_generator.generator.env import WORLDS_NAMES from maps_generator.generator.env import WORLD_NAME from maps_generator.generator.env import get_all_countries_list -from maps_generator.generator.exceptions import ValidationError +from maps_generator.generator.exceptions import ValidationError, wait_and_raise_if_fail from maps_generator.generator.gen_tool import run_gen_tool from maps_generator.generator.osmtools import osmconvert from maps_generator.generator.osmtools import osmupdate @@ -213,8 +213,7 @@ def step_prepare_routing_world(env: Env, country: AnyStr, **kwargs): env=os.environ ) - if sub_proc.wait() != 0: - raise Exception(f"Error running {world_roads_builder_tool_with_args}") + wait_and_raise_if_fail(sub_proc) def step_routing_world(env: Env, country: AnyStr, **kwargs): diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 7bb36bacf1..b9a2b7b377 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -59,7 +59,7 @@ DIFF_VERSION_DEPTH: 2 # PLANET_COASTS_URL: # Set to 'true' to build special routing section in World.mwm for alerting about absent regions without which the # route can't be built. -BUILD_WORLD_ROADS: true +NEED_BUILD_WORLD_ROADS: true # The url to the subway file. SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json From 6ebf6172496268b50f407c21323fd961acc5d4eb Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Mon, 11 Jan 2021 10:14:31 +0300 Subject: [PATCH 10/28] [generator] Import wait_and_raise_if_fail. --- tools/python/maps_generator/generator/steps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 52a85917b9..46fcfcbd6a 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -15,7 +15,8 @@ from maps_generator.generator.env import PathProvider from maps_generator.generator.env import WORLDS_NAMES from maps_generator.generator.env import WORLD_NAME from maps_generator.generator.env import get_all_countries_list -from maps_generator.generator.exceptions import ValidationError, wait_and_raise_if_fail +from maps_generator.generator.exceptions import ValidationError +from maps_generator.generator.exceptions import wait_and_raise_if_fail from maps_generator.generator.gen_tool import run_gen_tool from maps_generator.generator.osmtools import osmconvert from maps_generator.generator.osmtools import osmupdate From 7e879a37cacff00ab450ccbc6b8f9f5fe1723b15 Mon Sep 17 00:00:00 2001 From: Alexey Osminin Date: Mon, 11 Jan 2021 11:58:05 +0300 Subject: [PATCH 11/28] [android] fix for status bar when prompt is shown --- android/build.gradle | 4 +++- android/src/com/mapswithme/maps/tips/Tutorial.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index d070416f5f..346ff90db2 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -81,7 +81,9 @@ dependencies { implementation 'androidx.constraintlayout:constraintlayout:1.1.3' implementation 'androidx.work:work-runtime:2.2.0' implementation 'com.android.billingclient:billing:1.1' - implementation 'uk.co.samuelwall:material-tap-target-prompt:2.12.1' + implementation('uk.co.samuelwall:material-tap-target-prompt:3.1.0', { + exclude group: 'androidx.lifecycle', module: 'lifecycle-extensions' + }) implementation 'com.firebase:firebase-jobdispatcher:0.8.5' implementation 'com.google.android:flexbox:1.0.0' implementation 'com.trafi:anchor-bottom-sheet-behavior:0.13-alpha' diff --git a/android/src/com/mapswithme/maps/tips/Tutorial.java b/android/src/com/mapswithme/maps/tips/Tutorial.java index b2578024ed..660a82abe7 100644 --- a/android/src/com/mapswithme/maps/tips/Tutorial.java +++ b/android/src/com/mapswithme/maps/tips/Tutorial.java @@ -156,7 +156,8 @@ public enum Tutorial .setBackgroundColour(ThemeUtils.getColor(activity, R.attr.tipsBgColor)) .setFocalColour(activity.getResources().getColor(android.R.color.transparent)) .setPromptBackground(new ImmersiveModeCompatPromptBackground(activity.getWindowManager())) - .setPromptStateChangeListener(listener); + .setPromptStateChangeListener(listener) + .setIgnoreStatusBar(true); builder.show(); } From 1825dc3c18a0b331ec5ea119888cc37f49f83d1c Mon Sep 17 00:00:00 2001 From: Alexey Osminin Date: Tue, 12 Jan 2021 13:33:34 +0300 Subject: [PATCH 12/28] [android] fix for lost jni signature refactoring --- android/jni/com/mapswithme/platform/Platform.cpp | 11 ++++++----- android/src/com/mapswithme/util/BatteryState.java | 10 ++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/android/jni/com/mapswithme/platform/Platform.cpp b/android/jni/com/mapswithme/platform/Platform.cpp index af2ebd288c..f36bf641b4 100644 --- a/android/jni/com/mapswithme/platform/Platform.cpp +++ b/android/jni/com/mapswithme/platform/Platform.cpp @@ -127,9 +127,10 @@ Platform::ChargingStatus Platform::GetChargingStatus() ASSERT(clazzBatteryState, ()); static jmethodID const getChargingMethodId = - jni::GetStaticMethodID(env, clazzBatteryState, "getChargingStatus", "()I"); + jni::GetStaticMethodID(env, clazzBatteryState, "getChargingStatus", "(Landroid/content/Context;)I"); + jobject context = android::Platform::Instance().GetContext(); return static_cast( - env->CallStaticIntMethod(clazzBatteryState, getChargingMethodId)); + env->CallStaticIntMethod(clazzBatteryState, getChargingMethodId, context)); } uint8_t Platform::GetBatteryLevel() @@ -143,9 +144,9 @@ uint8_t Platform::GetBatteryLevel() ASSERT(clazzBatteryState, ()); static auto const getLevelMethodId = - jni::GetStaticMethodID(env, clazzBatteryState, "getLevel", "()I"); - - return static_cast(env->CallStaticIntMethod(clazzBatteryState, getLevelMethodId)); + jni::GetStaticMethodID(env, clazzBatteryState, "getLevel", "(Landroid/content/Context;)I"); + jobject context = android::Platform::Instance().GetContext(); + return static_cast(env->CallStaticIntMethod(clazzBatteryState, getLevelMethodId, context)); } namespace platform diff --git a/android/src/com/mapswithme/util/BatteryState.java b/android/src/com/mapswithme/util/BatteryState.java index ca39ebc50e..64be40c1d6 100644 --- a/android/src/com/mapswithme/util/BatteryState.java +++ b/android/src/com/mapswithme/util/BatteryState.java @@ -38,6 +38,8 @@ public final class BatteryState return new State(getLevel(batteryStatus), getChargingStatus(batteryStatus)); } + // Called from JNI. + @SuppressWarnings("unused") @IntRange(from=0, to=100) public static int getLevel(@NonNull Context context) { @@ -50,6 +52,14 @@ public final class BatteryState return batteryStatus.getIntExtra(BatteryManager.EXTRA_LEVEL, 0); } + // Called from JNI. + @SuppressWarnings("unused") + @ChargingStatus + public static int getChargingStatus(@NonNull Context context) + { + return getState(context).getChargingStatus(); + } + @ChargingStatus private static int getChargingStatus(@NonNull Intent batteryStatus) { From 3f66ec64a9418551d9b951fa616d4b7206e8d828 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 1 Jan 2021 23:13:37 +0300 Subject: [PATCH 13/28] [generator] Fixed countries.txt generation. --- .../generator/stages_declaration.py | 32 +++++++++++-------- tools/python/post_generation/__main__.py | 8 ++--- .../post_generation/hierarchy_to_countries.py | 2 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index 5fbbba0da3..d1aa76ae8b 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -112,7 +112,9 @@ class StagePreprocess(Stage): D(settings.FOOD_URL, PathProvider.food_paths, "p"), D(settings.FOOD_TRANSLATIONS_URL, PathProvider.food_translations_path, "p"), ) -@test_stage(Test(st.make_test_booking_data(max_days=7), lambda e, _: e.production, True)) +@test_stage( + Test(st.make_test_booking_data(max_days=7), lambda e, _: e.production, True) +) class StageFeatures(Stage): def apply(self, env: Env): extra = {} @@ -179,8 +181,13 @@ class StageMwm(Stage): @staticmethod def make_mwm(country: AnyStr, env: Env): world_stages = { - WORLD_NAME: [StageIndex, StageCitiesIdsWorld, StagePrepareRoutingWorld, StageRoutingWorld, - StageMwmStatistics], + WORLD_NAME: [ + StageIndex, + StageCitiesIdsWorld, + StagePrepareRoutingWorld, + StageRoutingWorld, + StageMwmStatistics, + ], WORLD_COASTS_NAME: [StageIndex, StageMwmStatistics], } @@ -302,12 +309,11 @@ class StageRoutingTransit(Stage): class StageMwmDiffs(Stage): def apply(self, env: Env, country, logger, **kwargs): data_dir = diffs.DataDir( - mwm_name=env.build_name, new_version_dir=env.build_path, - old_version_root_dir=settings.DATA_ARCHIVE_DIR - ) - diffs.mwm_diff_calculation( - data_dir, logger, depth=settings.DIFF_VERSION_DEPTH + mwm_name=env.build_name, + new_version_dir=env.build_path, + old_version_root_dir=settings.DATA_ARCHIVE_DIR, ) + diffs.mwm_diff_calculation(data_dir, logger, depth=settings.DIFF_VERSION_DEPTH) @country_stage @@ -323,7 +329,8 @@ class StageMwmStatistics(Stage): settings.PROMO_CATALOG_COUNTRIES_URL, PathProvider.promo_catalog_countries_path, "p", - ) + ), + D(settings.PROMO_CATALOG_CITIES_URL, PathProvider.promo_catalog_cities_path, "p"), ) class StageCountriesTxt(Stage): def apply(self, env: Env): @@ -336,9 +343,8 @@ class StageCountriesTxt(Stage): env.paths.mwm_version, ) if env.production: - countries_json = json.loads(countries) inject_promo_ids( - countries_json, + countries, env.paths.promo_catalog_cities_path, env.paths.promo_catalog_countries_path, env.paths.mwm_path, @@ -346,8 +352,8 @@ class StageCountriesTxt(Stage): env.paths.mwm_path, ) - with open(env.paths.counties_txt_path, "w") as f: - json.dump(countries_json, f, ensure_ascii=True, indent=1) + with open(env.paths.counties_txt_path, "w") as f: + json.dump(countries, f, ensure_ascii=True, indent=1) @outer_stage diff --git a/tools/python/post_generation/__main__.py b/tools/python/post_generation/__main__.py index 0b97f9647a..dd0a0a686a 100644 --- a/tools/python/post_generation/__main__.py +++ b/tools/python/post_generation/__main__.py @@ -90,7 +90,7 @@ The post_generation commands are: help="Output countries.txt file (default is stdout)", ) args = parser.parse_args(sys.argv[2:]) - countries_json = hierarchy_to_countries_( + countries = hierarchy_to_countries_( args.old, args.osm, args.countries_synonyms, @@ -100,9 +100,9 @@ The post_generation commands are: ) if args.output: with open(args.output, "w") as f: - f.write(countries_json) + json.dump(countries, f, ensure_ascii=True, indent=1) else: - print(countries_json) + print(json.dumps(countries, ensure_ascii=True, indent=1)) @staticmethod def inject_promo_ids(): @@ -152,7 +152,7 @@ The post_generation commands are: ) with open(args.output, "w") as f: - json.dump(countries, f, indent=1) + json.dump(countries, f, ensure_ascii=True, indent=1) PostGeneration() diff --git a/tools/python/post_generation/hierarchy_to_countries.py b/tools/python/post_generation/hierarchy_to_countries.py index 777f4f0079..9853580080 100755 --- a/tools/python/post_generation/hierarchy_to_countries.py +++ b/tools/python/post_generation/hierarchy_to_countries.py @@ -185,4 +185,4 @@ def hierarchy_to_countries( stack[-1]["g"].append(g) collapse_single(stack[-1]) - return json.dumps(stack[-1], ensure_ascii=True, indent=1) + return stack[-1] From 6cdfd4c960daaf9932d06d982e2d36612662dc58 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Wed, 6 Jan 2021 15:46:10 +0300 Subject: [PATCH 14/28] [generator] Fixed download_file() --- tools/python/maps_generator/utils/file.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/python/maps_generator/utils/file.py b/tools/python/maps_generator/utils/file.py index 1036d8b9c7..f97bcfae2a 100644 --- a/tools/python/maps_generator/utils/file.py +++ b/tools/python/maps_generator/utils/file.py @@ -54,7 +54,14 @@ def download_file(url: AnyStr, name: AnyStr, download_if_exists: bool = True): session.mount("file://", FileAdapter()) with open(tmp_name, "wb") as handle: response = session.get(url, stream=True) - file_length = int(response.headers["Content-Length"]) + file_length = None + try: + file_length = int(response.headers["Content-Length"]) + except KeyError: + logger.warning( + f"There is no attribute Content-Length in headers [{url}]: {response.headers}" + ) + current = 0 max_attempts = 32 attempts = max_attempts @@ -63,7 +70,7 @@ def download_file(url: AnyStr, name: AnyStr, download_if_exists: bool = True): current += len(data) handle.write(data) - if file_length == current: + if file_length is None or file_length == current: break logger.warning( From 35d0e83db8905b5e87af5f87b4ebce10af44b649 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Thu, 14 Jan 2021 14:16:42 +0300 Subject: [PATCH 15/28] [generator] Fixed adding duplicated types. --- generator/final_processor_country.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/generator/final_processor_country.cpp b/generator/final_processor_country.cpp index 645077de21..aae240e282 100644 --- a/generator/final_processor_country.cpp +++ b/generator/final_processor_country.cpp @@ -254,20 +254,18 @@ void CountryFinalProcessor::ProcessBuildingParts() m4::Tree buildingPartsKDTree; ForEachFeatureRawFormat(path, [&](auto && fb, auto /* pos */) { - if (!(fb.IsArea() && fb.IsValid())) - return; - - if (fb.HasType(buildingPartClassifType)) + if (fb.IsArea() && fb.HasType(buildingPartClassifType)) buildingPartsKDTree.Add(fb); }); FeatureBuilderWriter writer(path, true /* mangleName */); ForEachFeatureRawFormat(path, [&](auto && fb, auto /* pos */) { - if (fb.IsArea() && fb.IsValid() && + if (fb.IsArea() && fb.HasType(buildingClassifType) && DoesBuildingConsistOfParts(fb, buildingPartsKDTree)) { fb.AddType(buildingWithPartsClassifType); + fb.GetParams().FinishAddingTypes(); } writer.Write(fb); From 213a7bbb6827cb853b3b8349b198c96464017c2b Mon Sep 17 00:00:00 2001 From: Anatoliy Tomilov Date: Tue, 5 Jan 2021 04:37:01 +0300 Subject: [PATCH 16/28] [elbrus] Porting to e2k-v4 arch. Compiler version: lcc:1.25.10:Nov--7-2020:e2k-v4-linux --- openlr/router.cpp | 24 +++++++++++-------- .../routing_tests/followed_polyline_test.cpp | 8 +++---- search/geocoder.cpp | 8 +++---- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/openlr/router.cpp b/openlr/router.cpp index 342d18d052..98ec70b4cb 100644 --- a/openlr/router.cpp +++ b/openlr/router.cpp @@ -648,20 +648,22 @@ bool Router::ReconstructPath(std::vector & edges, std::vector & edges, std::vectorm_u, false /* outgoing */, m_points[0].m_lfrcnp, [&](routing::Edge const & edge) { + auto toPairRev = [](auto && e) { return e.ToPairRev(); }; double const score = GetMatchingScore( edge.GetEndJunction().GetPoint(), edge.GetStartJunction().GetPoint(), - make_transform_iterator(EdgeItRev(e), std::mem_fn(&Edge::ToPairRev)), - make_transform_iterator(edges.rend(), std::mem_fn(&Edge::ToPairRev))); + make_transform_iterator(EdgeItRev(e), toPairRev), + make_transform_iterator(edges.rend(), toPairRev)); if (score > frontEdgeScore) { frontEdgeScore = score; @@ -688,10 +691,11 @@ bool Router::ReconstructPath(std::vector & edges, std::vectorm_v, true /* outgoing */, m_points[m_points.size() - 2].m_lfrcnp, [&](routing::Edge const & edge) { + auto toPair = [](auto && e) { return e.ToPair(); }; double const score = GetMatchingScore( edge.GetStartJunction().GetPoint(), edge.GetEndJunction().GetPoint(), - make_transform_iterator(e.base(), std::mem_fn(&Edge::ToPair)), - make_transform_iterator(edges.end(), std::mem_fn(&Edge::ToPair))); + make_transform_iterator(e.base(), toPair), + make_transform_iterator(edges.end(), toPair)); if (score > backEdgeScore) { backEdgeScore = score; @@ -727,7 +731,7 @@ void Router::FindSingleEdgeApproximation(std::vector const & edges, { double const kCoverageThreshold = 0.5; - CHECK(all_of(edges.begin(), edges.end(), std::mem_fn(&Edge::IsFake)), ()); + CHECK(all_of(edges.begin(), edges.end(), [](auto && e) { return e.IsFake(); }), ()); double expectedLength = 0; for (auto const & edge : edges) diff --git a/routing/routing_tests/followed_polyline_test.cpp b/routing/routing_tests/followed_polyline_test.cpp index b9f9b889d1..77225ca0e0 100644 --- a/routing/routing_tests/followed_polyline_test.cpp +++ b/routing/routing_tests/followed_polyline_test.cpp @@ -10,8 +10,8 @@ using namespace routing; namespace { - static const m2::PolylineD kTestDirectedPolyline1({{0.0, 0.0}, {3.0, 0.0}, {5.0, 0.0}}); - static const m2::PolylineD kTestDirectedPolyline2({{6.0, 0.0}, {7.0, 0.0}}); + static const m2::PolylineD kTestDirectedPolyline1(std::vector{{0.0, 0.0}, {3.0, 0.0}, {5.0, 0.0}}); + static const m2::PolylineD kTestDirectedPolyline2(std::vector{{6.0, 0.0}, {7.0, 0.0}}); } // namespace UNIT_TEST(FollowedPolylineAppend) @@ -100,7 +100,7 @@ UNIT_TEST(FollowedPolylineDistanceCalculationTest) UNIT_TEST(FollowedPolylineDirectionTest) { - m2::PolylineD testPolyline({{0, 0}, {1.00003, 0}, {1.00003, 1}}); + m2::PolylineD testPolyline(std::vector{{0, 0}, {1.00003, 0}, {1.00003, 1}}); FollowedPolyline polyline(testPolyline.Begin(), testPolyline.End()); TEST_EQUAL(polyline.GetCurrentIter().m_ind, 0, ()); m2::PointD directionPoint; @@ -115,7 +115,7 @@ UNIT_TEST(FollowedPolylineDirectionTest) UNIT_TEST(FollowedPolylineGetDistanceFromBeginM) { - m2::PolylineD testPolyline({{0, 0}, {1, 0}, {2, 0}, {3, 0}, {5, 0}, {6, 0}}); + m2::PolylineD testPolyline(std::vector{{0, 0}, {1, 0}, {2, 0}, {3, 0}, {5, 0}, {6, 0}}); FollowedPolyline polyline(testPolyline.Begin(), testPolyline.End()); m2::PointD point(4, 0); polyline.UpdateProjection(mercator::RectByCenterXYAndSizeInMeters(point, 2)); diff --git a/search/geocoder.cpp b/search/geocoder.cpp index b54f3af383..34ef83db37 100644 --- a/search/geocoder.cpp +++ b/search/geocoder.cpp @@ -1223,7 +1223,7 @@ void Geocoder::GreedilyMatchStreetsWithSuburbs(BaseContext & ctx, auto & layers = ctx.m_layers; ASSERT(layers.empty(), ()); layers.emplace_back(); - SCOPE_GUARD(cleanupGuard, bind(&vector::pop_back, &layers)); + SCOPE_GUARD(cleanupGuard, [&]{ layers.pop_back(); }); auto & layer = layers.back(); InitLayer(Model::TYPE_SUBURB, suburb.m_tokenRange, layer); @@ -1262,7 +1262,7 @@ void Geocoder::CreateStreetsLayerAndMatchLowerLayers(BaseContext & ctx, auto & layers = ctx.m_layers; layers.emplace_back(); - SCOPE_GUARD(cleanupGuard, bind(&vector::pop_back, &layers)); + SCOPE_GUARD(cleanupGuard, [&]{ layers.pop_back(); }); auto & layer = layers.back(); InitLayer(Model::TYPE_STREET, prediction.m_tokenRange, layer); @@ -1353,7 +1353,7 @@ void Geocoder::MatchPOIsAndBuildings(BaseContext & ctx, size_t curToken, CBV con // Following code creates a fake layer with buildings and // intersects it with the streets layer. layers.emplace_back(); - SCOPE_GUARD(cleanupGuard, bind(&vector::pop_back, &layers)); + SCOPE_GUARD(cleanupGuard, [&]{ layers.pop_back(); }); auto & layer = layers.back(); InitLayer(Model::TYPE_BUILDING, m_postcodes.m_tokenRange, layer); @@ -1367,7 +1367,7 @@ void Geocoder::MatchPOIsAndBuildings(BaseContext & ctx, size_t curToken, CBV con } layers.emplace_back(); - SCOPE_GUARD(cleanupGuard, bind(&vector::pop_back, &layers)); + SCOPE_GUARD(cleanupGuard, [&]{ layers.pop_back(); }); // Clusters of features by search type. Each cluster is a sorted // list of ids. From 1b4e8ca9ff09f584abc9d412b406bb48ea6342a5 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Wed, 30 Dec 2020 14:55:50 +0300 Subject: [PATCH 17/28] [README] Instructions for turning on full public transport support. --- docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md diff --git a/docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md b/docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md new file mode 100644 index 0000000000..1141100662 --- /dev/null +++ b/docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md @@ -0,0 +1,81 @@ +# Experimental support of public transport from GTFS + +## Introduction to the experimental public transport feature + +At the moment our app does not have full support for public transport. What we have now: + +- Scarce transit data collected from OSM which includes subway, light rail, monorail and train routes and stops. Let's call it the [OSM transit](https://github.com/mapsme/omim/blob/master/docs/SUBWAY_GENERATION.md) from now on. +- The "Subway layer" for visual representation of the OSM transit. +- "Subway navigation" - OSM transit routing. + + +:bus: But you can turn on the experimental feature of [GTFS](https://developers.google.com/transit/gtfs/reference) public transport and use it inside the MAPS.ME app. It includes all [transit types definded by GTFS specification:](https://developers.google.com/transit/gtfs/reference/extended-route-types) bus, train, ferry, aerial lifts of different kinds, trolleybus and much more. Let's call this version of transit data **GTFS transit** from now on. + +To mixin the experimental GTFS transit into the OSM transit data you should follow 2 simple steps: +- Run the pipeline for downloading and preparing GTFS data about public transport. +- Switch to the new version of the transit routing section in maps: build maps with the GTFS transit section with the help of special options for generator_tool. + +After completing these steps you will have maps with: + +:star: Proper public transport navigation. You'll have the opportunity to build public transport routes with transfers from one type of public transport to another (e.g. bus -> train, monorail -> subway, etc). The route is built according to the transit schedules, days off and exceptions in timetables. + +:star: "Subway layer" with old OSM transit data but in new visual representation. It differs from the current layer in handling parallel segments of different routes with the same color. In current version we draw separate polyline for each route. It looks a little bit messy, and the new version fixes this: we collapse single-colored parallel routes into one line. + +Most of the data collisions between OSM and GTFS sources are excluded, because we filter all subway data from GTFS on purpose. There still could be collisions in light rail, monorail and train data, but it is not a problem: on the "Subway layer" we show data only from OSM. In routing we choose optimal route from GTFS and OSM. + + +**Why is this feature experimental?** The main reason is that it is not properly tested. Also it lacks new user interfaces for filtering public transport by type, setting departure time and routing "follow mode". + + +## Instructions for turning on full public transport support + +1. Run the python script for collecting GTFS feeds from two GTFS aggregators: [Transitland](https://www.transit.land/) and [OpenMobilityData.](http://transitfeeds.com/feeds) You can find the script in omim/tools/python/transit/gtfs/. It is called download_gtfs.py. It crawls all the feeds from these aggregators, loads feed zips and extracts as subdirectories to the specified directory. + +Example: +``` +python3 download_gtfs.py --path=dir_for_storing_feeds --mode=fullrun --source=all --omd_api_key=YOUR_KEY_FOR_OPEN_MOBILITY_DATA_API +``` + +In this example all accessible feeds from Transitland and OpenMobilityData will be downloaded to the `dir_for_storing_feeds` directory. But if you don't want to create key for OpenMobilityData API you can run this tool for crawling Transitland only: + +``` +python3 download_gtfs.py --path=dir_for_storing_feeds --mode=fullrun --source=transitland +``` + +After completing this step you will have directory containing subdirectories with GTFS feeds. + +2. Build and run the C++ tool for filtering, combining and transforming GTFS feeds (downloaded on the previous step) to the intermediate json files. Later they will be passed to the generator_tool for creating transit section in new format. You can find the tool in `omim/transit/world_feed/gtfs_converter.` The tool is` gtfs_converter.` + +It can be run in two modes: for handling GTFS-only or for merging data from GTFS and OSM. + + +Example of running `gtfs_converter` for merging GTFS and OSM data: + +``` +./gtfs_converter --path_mapping=mapping.txt --path_mapping_edges=mapping_edges.txt --path_gtfs_feeds=dir_for_storing_feeds --path_subway_json=subway_latest.json --path_json=result_json_dir --path_resources=omim/data +``` + +Example of running `gtfs_converter` for saving only GTFS data (without OSM): + +``` +./gtfs_converter --path_mapping=mapping.txt --path_mapping_edges=mapping_edges.txt --path_gtfs_feeds=dir_for_storing_feeds --path_json=result_json_dir --path_resources=omim/data +``` + +:exclamation: GTFS data often changes: some routes are cancelled, some stops are built, timetables constantly change. **If you're planning to build maps periodically, more than once, then you must take care of the safety of files with transit entities ids which are generated by `gtfs_converter.`** The goal is to make transit entities consistent between re-runs of the tool so that routing between old regions and the new ones doesn't fail. It is better to backup files specified as `path_mapping` and `path_mapping_edges`. When you run `gtfs_converter` the first time these files are created. Subsequent launches update existing files. + + +After completing this step you will have directory with line-by-line jsons in the directory specified by `path_json` option of `gtfs_converter.` + + +3. In map_generator.ini uncomment `SUBWAY_URL` parameter and set `TRANSIT_URL` parameter to the directory created on the previous step. Example: +``` +TRANSIT_URL: file:///home/result_json_dir +``` + +Run generator tool [as usual](https://github.com/mapsme/omim/tree/master/tools/python/maps_generator) with this ini config. After it is done you'll have mwms with transit section in experimental GTFS format. + +:checkered_flag: Use the resulting mwms in your app. Enjoy the experimental public transport in MAPS.ME! + + +## If you have questions +Feel free to ask [mesozoic-drones](https://github.com/mesozoic-drones), you can also do it by mail: mesozoic.drones@gmail.com From 6ef8146fb19bb3cf5bfaaf3c994d5a010031730c Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Tue, 12 Jan 2021 12:17:09 +0300 Subject: [PATCH 18/28] [readme] Link from README to EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1ac3c22100..6e0767d3f0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ repository: see `3party` directory. The team uses mostly XCode and Qt Creator, though these are not mandatory. We have an established [c++ coding style](https://github.com/mapsme/omim/blob/master/docs/CPP_STYLE.md) and [Objective-C coding style](https://github.com/mapsme/omim/blob/master/docs/OBJC_STYLE.md). +**You can turn on experimental public transport support.** For details please read [simple instruction.](https://github.com/mapsme/omim/blob/master/docs/EXPERIMENTAL_PUBLIC_TRANSPORT_SUPPORT.md) + See [CONTRIBUTING.md](https://github.com/mapsme/omim/blob/master/docs/CONTRIBUTING.md) for the repository initialization process, the description of all the directories of this repository and other development-related information. From e20bdee2d1fa63fd079ce8bde837c9dec3c72c39 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Wed, 13 Jan 2021 10:43:08 +0300 Subject: [PATCH 19/28] [tools][transit] Handle exceptions while crawling Transitland & refactor log messages. --- tools/python/transit/gtfs/download_gtfs.py | 49 ++++++++++++---------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index ac18c55ce8..e1d1219932 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -35,9 +35,6 @@ def get_feeds_links(data): for feed in data: if feed["feed_format"] != "gtfs" or feed["spec"] != "gtfs": - # Warning about strange format - not gtfs and not real-time gtfs: - if feed["feed_format"] != "gtfs-rt": - logger.warning(f"Skipped feed: feed_format {feed['feed_format']}, spec {feed['spec']}") continue if "url" in feed and feed["url"] is not None and feed["url"]: @@ -51,25 +48,30 @@ def parse_transitland_page(url): retries = MAX_RETRIES while retries > 0: - with requests.get(url) as response: - if response.status_code != 200: - logger.error(f"Failed loading feeds: {response.status_code}") - if response.status_code == 429: - logger.error("Too many requests.") - time.sleep(MAX_SLEEP_TIMEOUT_S) + try: + with requests.get(url) as response: + if response.status_code != 200: + logger.error(f"Failed loading feeds: {response.status_code}") + if response.status_code == 429: + logger.error("Too many requests.") + time.sleep(MAX_SLEEP_TIMEOUT_S) + else: + time.sleep(AVG_SLEEP_TIMEOUT_S) + retries -= 1 + continue + + data = json.loads(response.text) + if "feeds" in data: + gtfs_feeds_urls = get_feeds_links(data["feeds"]) else: - time.sleep(AVG_SLEEP_TIMEOUT_S) - retries -= 1 - continue + gtfs_feeds_urls = [] - data = json.loads(response.text) - if "feeds" in data: - gtfs_feeds_urls = get_feeds_links(data["feeds"]) - else: - gtfs_feeds_urls = [] - - next_page = data["meta"]["next"] if "next" in data["meta"] else "" - return gtfs_feeds_urls, next_page + next_page = data["meta"]["next"] if "next" in data["meta"] else "" + return gtfs_feeds_urls, next_page + except requests.exceptions.RequestException as ex: + logger.error(f"Exception {ex} for url {url}") + retries -= 1 + time.sleep(AVG_SLEEP_TIMEOUT_S) return [], "" @@ -82,7 +84,10 @@ def extract_to_path(content, out_path): return True except zipfile.BadZipfile: logger.exception("BadZipfile exception.") - return False + except Exception as e: + logger.exception(f"Exception while unzipping feed: {e}") + + return False def load_gtfs_feed_zip(path, url): @@ -137,7 +142,7 @@ def parse_openmobilitydata_pages(omd_api_key): if page == 1: pages_count = data["results"]["numPages"] - logger.info(f"Pages count {pages_count}") + logger.info(f"There are {pages_count} Openmobilitydata pages with feed urls.") for feed in data["results"]["feeds"]: params = { From ea7df99cdd602f6808788b5b0ecd5404581c1f3e Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Wed, 13 Jan 2021 14:12:14 +0300 Subject: [PATCH 20/28] [transit] Handle exceptions in parse_openmobilitydata_pages(). --- tools/python/transit/gtfs/download_gtfs.py | 50 ++++++++++++---------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index e1d1219932..2c6b3eb2b4 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -133,32 +133,38 @@ def parse_openmobilitydata_pages(omd_api_key): "type": "gtfs" } - with requests.get(url_page, params=params, headers=HEADERS_OMD) as response: - if response.status_code != 200: - logger.error(f"Http code {response.status_code} loading feed ids: {url_page}") - return [], "" + try: + with requests.get(url_page, params=params, headers=HEADERS_OMD) as response: + if response.status_code != 200: + logger.error(f"Http code {response.status_code} loading feed ids: {url_page}") + return [], "" - data = json.loads(response.text) + data = json.loads(response.text) - if page == 1: - pages_count = data["results"]["numPages"] - logger.info(f"There are {pages_count} Openmobilitydata pages with feed urls.") + if page == 1: + pages_count = data["results"]["numPages"] + logger.info(f"There are {pages_count} Openmobilitydata pages with feed urls.") - for feed in data["results"]["feeds"]: - params = { - "key": omd_api_key, - "feed": feed["id"] - } + for feed in data["results"]["feeds"]: + params = { + "key": omd_api_key, + "feed": feed["id"] + } + try: + with requests.get(url_with_redirect, params=params, headers=HEADERS_OMD, allow_redirects=True) \ + as response_redirect: + if response_redirect.history: + urls.append(response_redirect.url) + else: + logger.error(f"Could not get link to zip with feed {feed['id']} from {url_with_redirect}") + except requests.exceptions.RequestException as ex_redirect: + logger.error(f"Exception {ex_redirect} while getting link to zip with " + f"feed {feed['id']} from {url_with_redirect}") + except requests.exceptions.RequestException as ex: + logger.error(f"Exception {ex} while getting {url_page} (page {page}) from Openmobilitydata.") - with requests.get(url_with_redirect, params=params, headers=HEADERS_OMD, allow_redirects=True) \ - as response_redirect: - if response_redirect.history: - urls.append(response_redirect.url) - else: - logger.error(f"Could not get link to zip with feed {feed['id']} from {url_with_redirect}") - - logger.info(f"page {page}") - page += 1 + logger.info(f"Crawled {page}/{pages_count} page of Openmobilitydata.") + page += 1 return urls From 0b864576479d39ef13f72dde1a5f9f40b417529c Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Fri, 15 Jan 2021 16:43:48 +0300 Subject: [PATCH 21/28] [transit] Reformatted code. --- tools/python/transit/gtfs/download_gtfs.py | 104 +++++++++++++++------ 1 file changed, 74 insertions(+), 30 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index 2c6b3eb2b4..8f883d086b 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -130,38 +130,49 @@ def parse_openmobilitydata_pages(omd_api_key): "location": "undefined", "descendants": 1, "limit": 100, - "type": "gtfs" + "type": "gtfs", } try: with requests.get(url_page, params=params, headers=HEADERS_OMD) as response: if response.status_code != 200: - logger.error(f"Http code {response.status_code} loading feed ids: {url_page}") + logger.error( + f"Http code {response.status_code} loading feed ids: {url_page}" + ) return [], "" data = json.loads(response.text) if page == 1: pages_count = data["results"]["numPages"] - logger.info(f"There are {pages_count} Openmobilitydata pages with feed urls.") + logger.info( + f"There are {pages_count} Openmobilitydata pages with feed urls." + ) for feed in data["results"]["feeds"]: - params = { - "key": omd_api_key, - "feed": feed["id"] - } + params = {"key": omd_api_key, "feed": feed["id"]} try: - with requests.get(url_with_redirect, params=params, headers=HEADERS_OMD, allow_redirects=True) \ - as response_redirect: + with requests.get( + url_with_redirect, + params=params, + headers=HEADERS_OMD, + allow_redirects=True, + ) as response_redirect: if response_redirect.history: urls.append(response_redirect.url) else: - logger.error(f"Could not get link to zip with feed {feed['id']} from {url_with_redirect}") + logger.error( + f"Could not get link to zip with feed {feed['id']} from {url_with_redirect}" + ) except requests.exceptions.RequestException as ex_redirect: - logger.error(f"Exception {ex_redirect} while getting link to zip with " - f"feed {feed['id']} from {url_with_redirect}") + logger.error( + f"Exception {ex_redirect} while getting link to zip with " + f"feed {feed['id']} from {url_with_redirect}" + ) except requests.exceptions.RequestException as ex: - logger.error(f"Exception {ex} while getting {url_page} (page {page}) from Openmobilitydata.") + logger.error( + f"Exception {ex} while getting {url_page} (page {page}) from Openmobilitydata." + ) logger.info(f"Crawled {page}/{pages_count} page of Openmobilitydata.") page += 1 @@ -197,7 +208,9 @@ def crawl_transitland_for_feed_urls(out_path): def get_filename(file_prefix, index): index_str = str(index) index_len = len(index_str) - zeroes_prefix = "" if MAX_INDEX_LEN < index_len else "0" * (MAX_INDEX_LEN - index_len) + zeroes_prefix = ( + "" if MAX_INDEX_LEN < index_len else "0" * (MAX_INDEX_LEN - index_len) + ) return file_prefix + "_" + zeroes_prefix + index_str @@ -211,11 +224,18 @@ def load_gtfs_zips_from_urls(path, urls_file, threads_count, file_prefix): err_count = 0 with concurrent.futures.ThreadPoolExecutor(max_workers=threads_count) as executor: - future_to_url = {executor.submit(load_gtfs_feed_zip, - os.path.join(path, get_filename(file_prefix, i)), url): - url for i, url in enumerate(urls)} + future_to_url = { + executor.submit( + load_gtfs_feed_zip, + os.path.join(path, get_filename(file_prefix, i)), + url, + ): url + for i, url in enumerate(urls) + } - for j, future in enumerate(concurrent.futures.as_completed(future_to_url), start=1): + for j, future in enumerate( + concurrent.futures.as_completed(future_to_url), start=1 + ): url = future_to_url[future] loaded = future.result() @@ -238,25 +258,45 @@ def main(): """Downloads urls of feeds from feed aggregators and saves to the file. Downloads feeds from these urls and saves to the directory.""" - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) parser.add_argument("-p", "--path", required=True, help="working directory path") - parser.add_argument("-m", "--mode", required=True, - help="fullrun | load_feed_urls | load_feed_zips") + parser.add_argument( + "-m", "--mode", required=True, help="fullrun | load_feed_urls | load_feed_zips" + ) - parser.add_argument("-s", "--source", default="transitland", - help="source of feeds: transitland | openmobilitydata | all") + parser.add_argument( + "-s", + "--source", + default="transitland", + help="source of feeds: transitland | openmobilitydata | all", + ) - parser.add_argument("-t", "--threads", type=int, default=THREADS_COUNT, - help="threads count for loading zips") + parser.add_argument( + "-t", + "--threads", + type=int, + default=THREADS_COUNT, + help="threads count for loading zips", + ) - parser.add_argument("-k", "--omd_api_key", default="", - help="user key for working with openmobilitydata API") + parser.add_argument( + "-k", + "--omd_api_key", + default="", + help="user key for working with openmobilitydata API", + ) args = parser.parse_args() - logging.basicConfig(filename=os.path.join(args.path, "crawling.log"), filemode="w", level=logging.INFO) + logging.basicConfig( + filename=os.path.join(args.path, "crawling.log"), + filemode="w", + level=logging.INFO, + ) if args.mode in ["fullrun", "load_feed_urls"]: @@ -264,14 +304,18 @@ def main(): crawl_transitland_for_feed_urls(args.path) if args.source in ["all", "openmobilitydata"]: if not args.omd_api_key: - logger.error("No key provided for openmobilitydata. Set omd_api_key argument.") + logger.error( + "No key provided for openmobilitydata. Set omd_api_key argument." + ) return crawl_openmobilitydata_for_feed_urls(args.path, args.omd_api_key) if args.mode in ["fullrun", "load_feed_zips"]: if args.source in ["all", "transitland"]: - load_gtfs_zips_from_urls(args.path, URLS_FILE_TRANSITLAND, args.threads, "tl") + load_gtfs_zips_from_urls( + args.path, URLS_FILE_TRANSITLAND, args.threads, "tl" + ) if args.source in ["all", "openmobilitydata"]: load_gtfs_zips_from_urls(args.path, URLS_FILE_OMD, args.threads, "omd") From 355185ecfa299ed4b2aea60a03871fbdfdbb98a1 Mon Sep 17 00:00:00 2001 From: Olga Khlopkova Date: Fri, 15 Jan 2021 17:23:19 +0300 Subject: [PATCH 22/28] [transit] raise_for_status(). --- tools/python/transit/gtfs/download_gtfs.py | 30 ++++++++++------------ 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index 8f883d086b..94adadce6c 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -50,15 +50,7 @@ def parse_transitland_page(url): while retries > 0: try: with requests.get(url) as response: - if response.status_code != 200: - logger.error(f"Failed loading feeds: {response.status_code}") - if response.status_code == 429: - logger.error("Too many requests.") - time.sleep(MAX_SLEEP_TIMEOUT_S) - else: - time.sleep(AVG_SLEEP_TIMEOUT_S) - retries -= 1 - continue + response.raise_for_status() data = json.loads(response.text) if "feeds" in data: @@ -69,9 +61,15 @@ def parse_transitland_page(url): next_page = data["meta"]["next"] if "next" in data["meta"] else "" return gtfs_feeds_urls, next_page except requests.exceptions.RequestException as ex: - logger.error(f"Exception {ex} for url {url}") + logger.error( + f"Exception {ex} while parsing Transitland url {url} with code {response.status_code}" + ) + if response.status_code == 429: + logger.error("Too many requests.") + time.sleep(MAX_SLEEP_TIMEOUT_S) + else: + time.sleep(AVG_SLEEP_TIMEOUT_S) retries -= 1 - time.sleep(AVG_SLEEP_TIMEOUT_S) return [], "" @@ -96,11 +94,7 @@ def load_gtfs_feed_zip(path, url): while retries > 0: try: with requests.get(url, stream=True) as response: - if response.status_code != 200: - logger.error(f"HTTP code {response.status_code} loading gtfs {url}") - retries -= 1 - time.sleep(MAX_SLEEP_TIMEOUT_S) - continue + response.raise_for_status() if not extract_to_path(response.content, path): retries -= 1 @@ -110,7 +104,9 @@ def load_gtfs_feed_zip(path, url): return True except requests.exceptions.RequestException as ex: - logger.error(f"Exception {ex} for url {url}") + logger.error( + f"Exception {ex} downloading zip from url {url}, HTTP code {response.status_code}" + ) retries -= 1 time.sleep(AVG_SLEEP_TIMEOUT_S) From 3428ee9d9e1ec2dca770a6654b4de658de104385 Mon Sep 17 00:00:00 2001 From: Khlopkova Olga Date: Tue, 19 Jan 2021 12:12:10 +0300 Subject: [PATCH 23/28] [tools][transit] Fixes & improvements for GTFS crawler. --- tools/python/transit/gtfs/download_gtfs.py | 60 +++++++++++----------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index 94adadce6c..f387e31268 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -14,8 +14,7 @@ import zipfile import requests -MAX_RETRIES = 3 -AVG_SLEEP_TIMEOUT_S = 10 +MAX_RETRIES = 2 MAX_SLEEP_TIMEOUT_S = 30 URLS_FILE_TRANSITLAND = "feed_urls_transitland.txt" @@ -49,27 +48,28 @@ def parse_transitland_page(url): while retries > 0: try: - with requests.get(url) as response: - response.raise_for_status() + response = requests.get(url) + response.raise_for_status() - data = json.loads(response.text) - if "feeds" in data: - gtfs_feeds_urls = get_feeds_links(data["feeds"]) - else: - gtfs_feeds_urls = [] + data = json.loads(response.text) + if "feeds" in data: + gtfs_feeds_urls = get_feeds_links(data["feeds"]) + else: + gtfs_feeds_urls = [] - next_page = data["meta"]["next"] if "next" in data["meta"] else "" - return gtfs_feeds_urls, next_page + next_page = data["meta"]["next"] if "next" in data["meta"] else "" + return gtfs_feeds_urls, next_page + + except requests.exceptions.HTTPError as http_err: + logger.error(f"HTTP error {http_err} downloading zip from url {url}") + if http_err == 429: + time.sleep(MAX_SLEEP_TIMEOUT_S) except requests.exceptions.RequestException as ex: logger.error( f"Exception {ex} while parsing Transitland url {url} with code {response.status_code}" ) - if response.status_code == 429: - logger.error("Too many requests.") - time.sleep(MAX_SLEEP_TIMEOUT_S) - else: - time.sleep(AVG_SLEEP_TIMEOUT_S) - retries -= 1 + + retries -= 1 return [], "" @@ -93,22 +93,23 @@ def load_gtfs_feed_zip(path, url): retries = MAX_RETRIES while retries > 0: try: - with requests.get(url, stream=True) as response: - response.raise_for_status() + response = requests.get(url, stream=True) + response.raise_for_status() - if not extract_to_path(response.content, path): - retries -= 1 - logger.error(f"Could not extract zip: {url}") - continue + if not extract_to_path(response.content, path): + retries -= 1 + logger.error(f"Could not extract zip: {url}") + continue - return True + return True + except requests.exceptions.HTTPError as http_err: + logger.error(f"HTTP error {http_err} downloading zip from url {url}") except requests.exceptions.RequestException as ex: logger.error( - f"Exception {ex} downloading zip from url {url}, HTTP code {response.status_code}" + f"Exception {ex} downloading zip from url {url}" ) - retries -= 1 - time.sleep(AVG_SLEEP_TIMEOUT_S) + retries -= 1 return False @@ -202,12 +203,11 @@ def crawl_transitland_for_feed_urls(out_path): def get_filename(file_prefix, index): - index_str = str(index) - index_len = len(index_str) + index_len = len(str(index)) zeroes_prefix = ( "" if MAX_INDEX_LEN < index_len else "0" * (MAX_INDEX_LEN - index_len) ) - return file_prefix + "_" + zeroes_prefix + index_str + return f"{file_prefix}_{zeroes_prefix}{index}" def load_gtfs_zips_from_urls(path, urls_file, threads_count, file_prefix): From 976e128b50a84a3d31d39cc3d8850b051697a9b5 Mon Sep 17 00:00:00 2001 From: Khlopkova Olga Date: Tue, 19 Jan 2021 12:14:14 +0300 Subject: [PATCH 24/28] [tools][transit] Reformatted log message. --- tools/python/transit/gtfs/download_gtfs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index f387e31268..9e1716439a 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -106,9 +106,7 @@ def load_gtfs_feed_zip(path, url): except requests.exceptions.HTTPError as http_err: logger.error(f"HTTP error {http_err} downloading zip from url {url}") except requests.exceptions.RequestException as ex: - logger.error( - f"Exception {ex} downloading zip from url {url}" - ) + logger.error(f"Exception {ex} downloading zip from url {url}") retries -= 1 return False From 2626bb0f283b9c8e00ad3164fe51e202082ad4c9 Mon Sep 17 00:00:00 2001 From: Khlopkova Olga Date: Tue, 19 Jan 2021 12:28:17 +0300 Subject: [PATCH 25/28] [tools][transit] Fixes. --- tools/python/transit/gtfs/download_gtfs.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tools/python/transit/gtfs/download_gtfs.py b/tools/python/transit/gtfs/download_gtfs.py index 9e1716439a..ac895cd1bf 100644 --- a/tools/python/transit/gtfs/download_gtfs.py +++ b/tools/python/transit/gtfs/download_gtfs.py @@ -61,7 +61,7 @@ def parse_transitland_page(url): return gtfs_feeds_urls, next_page except requests.exceptions.HTTPError as http_err: - logger.error(f"HTTP error {http_err} downloading zip from url {url}") + logger.error(f"HTTP error {http_err} downloading zip from {url}") if http_err == 429: time.sleep(MAX_SLEEP_TIMEOUT_S) except requests.exceptions.RequestException as ex: @@ -104,9 +104,9 @@ def load_gtfs_feed_zip(path, url): return True except requests.exceptions.HTTPError as http_err: - logger.error(f"HTTP error {http_err} downloading zip from url {url}") + logger.error(f"HTTP error {http_err} downloading zip from {url}") except requests.exceptions.RequestException as ex: - logger.error(f"Exception {ex} downloading zip from url {url}") + logger.error(f"Exception {ex} downloading zip from {url}") retries -= 1 return False @@ -201,11 +201,7 @@ def crawl_transitland_for_feed_urls(out_path): def get_filename(file_prefix, index): - index_len = len(str(index)) - zeroes_prefix = ( - "" if MAX_INDEX_LEN < index_len else "0" * (MAX_INDEX_LEN - index_len) - ) - return f"{file_prefix}_{zeroes_prefix}{index}" + return f"{file_prefix}_{index:0{MAX_INDEX_LEN}d}" def load_gtfs_zips_from_urls(path, urls_file, threads_count, file_prefix): From d5065e7817deac6b05278cbfe447732b18fc38b1 Mon Sep 17 00:00:00 2001 From: velichkomarija Date: Tue, 19 Jan 2021 12:12:32 +0300 Subject: [PATCH 26/28] [android] Fix empty distance when user go straight. --- .../maps/routing/NavigationController.java | 30 +++++++++---------- .../mapswithme/maps/routing/RoutingInfo.java | 2 -- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/android/src/com/mapswithme/maps/routing/NavigationController.java b/android/src/com/mapswithme/maps/routing/NavigationController.java index d35df1e4e0..12b01b9074 100644 --- a/android/src/com/mapswithme/maps/routing/NavigationController.java +++ b/android/src/com/mapswithme/maps/routing/NavigationController.java @@ -6,8 +6,8 @@ import android.content.Context; import android.content.Intent; import android.location.Location; import android.media.MediaPlayer; -import android.os.Build; import android.os.Bundle; +import android.text.SpannableStringBuilder; import android.text.TextUtils; import android.util.Pair; import android.view.View; @@ -90,14 +90,7 @@ public class NavigationController implements TrafficManager.TrafficCallback, Vie { mFrame = activity.findViewById(R.id.navigation_frame); mBottomFrame = mFrame.findViewById(R.id.nav_bottom_frame); - mBottomFrame.setOnClickListener(new View.OnClickListener() - { - @Override - public void onClick(View v) - { - switchTimeFormat(); - } - }); + mBottomFrame.setOnClickListener(v -> switchTimeFormat()); mNavMenu = createNavMenu(); mNavMenu.refresh(); @@ -114,7 +107,7 @@ public class NavigationController implements TrafficManager.TrafficCallback, Vie mStreetFrame = topFrame.findViewById(R.id.street_frame); mNextStreet = (TextView) mStreetFrame.findViewById(R.id.street); View shadow = topFrame.findViewById(R.id.shadow_top); - UiUtils.showIf(Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP, shadow); + UiUtils.show(shadow); UiUtils.extendViewWithStatusBar(mStreetFrame); UiUtils.extendViewMarginWithStatusBar(turnFrame); @@ -196,12 +189,17 @@ public class NavigationController implements TrafficManager.TrafficCallback, Vie private void updateVehicle(RoutingInfo info) { - mNextTurnDistance.setText(Utils.formatUnitsText(mFrame.getContext(), - R.dimen.text_size_nav_number, - R.dimen.text_size_nav_dimension, - info.distToTurn, - info.turnUnits)); - info.carDirection.setTurnDrawable(mNextTurnImage); + if (!TextUtils.isEmpty(info.distToTurn)) + { + SpannableStringBuilder nextTurnDistance = Utils.formatUnitsText(mFrame.getContext(), + R.dimen.text_size_nav_number, + R.dimen.text_size_nav_dimension, + info.distToTurn, + info.turnUnits); + mNextTurnDistance.setText(nextTurnDistance); + info.carDirection.setTurnDrawable(mNextTurnImage); + } + if (RoutingInfo.CarDirection.isRoundAbout(info.carDirection)) UiUtils.setTextAndShow(mCircleExit, String.valueOf(info.exitNum)); else diff --git a/android/src/com/mapswithme/maps/routing/RoutingInfo.java b/android/src/com/mapswithme/maps/routing/RoutingInfo.java index 789ae9c9d9..2189279066 100644 --- a/android/src/com/mapswithme/maps/routing/RoutingInfo.java +++ b/android/src/com/mapswithme/maps/routing/RoutingInfo.java @@ -1,13 +1,11 @@ package com.mapswithme.maps.routing; -import android.location.Location; import androidx.annotation.DrawableRes; import androidx.annotation.NonNull; import android.widget.ImageView; import com.mapswithme.maps.R; -import com.mapswithme.maps.bookmarks.data.DistanceAndAzimut; public class RoutingInfo { From 8dc3e29730a2ce3ebd21ed8026e8fadb20519026 Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Tue, 19 Jan 2021 17:16:44 +0300 Subject: [PATCH 27/28] [transit] Minor refactoring: move subway checks to single place. --- transit/world_feed/feed_helpers.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/transit/world_feed/feed_helpers.cpp b/transit/world_feed/feed_helpers.cpp index 82e34582ca..f88299d7ad 100644 --- a/transit/world_feed/feed_helpers.cpp +++ b/transit/world_feed/feed_helpers.cpp @@ -159,13 +159,19 @@ std::pair PrepareNearestPointOnTrack(m2::PointD const & point, return {proj->m_indexOnShape, proj->m_needsInsertion}; } -bool IsRelevantType(const gtfs::RouteType & routeType) +bool IsRelevantType(gtfs::RouteType const & routeType) { // All types and constants are described in GTFS: // https://developers.google.com/transit/gtfs/reference + auto const isSubway = [](gtfs::RouteType const & routeType) { + return routeType == gtfs::RouteType::Subway || + routeType == gtfs::RouteType::MetroService || + routeType == gtfs::RouteType::UndergroundService; + }; + // We skip all subways because we extract subway data from OSM, not from GTFS. - if (routeType == gtfs::RouteType::Subway) + if (isSubway(routeType)) return false; auto const val = static_cast(routeType); @@ -183,8 +189,6 @@ bool IsRelevantType(const gtfs::RouteType & routeType) gtfs::RouteType::CarTransportRailService, gtfs::RouteType::LorryTransportRailService, gtfs::RouteType::VehicleTransportRailService, - gtfs::RouteType::MetroService, - gtfs::RouteType::UndergroundService, gtfs::RouteType::PostBusService, gtfs::RouteType::SpecialNeedsBus, gtfs::RouteType::MobilityBusService, From 0df8e92640115f6793e07b569114c7b5c0698b8e Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Wed, 20 Jan 2021 14:53:24 +0300 Subject: [PATCH 28/28] [just_gtfs] Update just_gtfs version. --- 3party/just_gtfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3party/just_gtfs b/3party/just_gtfs index d8510d0a4e..0b3559b70b 160000 --- a/3party/just_gtfs +++ b/3party/just_gtfs @@ -1 +1 @@ -Subproject commit d8510d0a4e9893a9ff6fc0521477773fdf991905 +Subproject commit 0b3559b70b183ddbf6fd7355bbfc62a383d431e4