diff --git a/tools/python/maps_generator/README.md b/tools/python/maps_generator/README.md index 3de4d475ed..44becdfae1 100644 --- a/tools/python/maps_generator/README.md +++ b/tools/python/maps_generator/README.md @@ -74,6 +74,12 @@ OSM_TOOLS_PATH: ~/osmctools [Stages] # Run osmupdate tool for planet. NEED_PLANET_UPDATE: 0 +# Auto detection. +THREADS_COUNT_FEATURES_STAGE: 0 +# If you want to calculate diffs, you need to specify, where old maps are +DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH} +# You may specify, how many versions in the archive to use for diff calculation +DIFF_VERSION_DEPTH: 2 [Logging] diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index ba036dec2d..8ca653994e 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -92,15 +92,14 @@ OSM_TOOLS_SRC_PATH = os.path.join(OMIM_PATH, "tools", "osmctools") OSM_TOOLS_PATH = os.path.join(_WORK_PATH, "osmctools") # Generator tool section: -NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" - USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data") - -DATA_ARCHIVE_DIR = USER_RESOURCE_PATH -DIFF_VERSION_DEPTH = 2 +NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" # Stages section: NEED_PLANET_UPDATE = False +THREADS_COUNT_FEATURES_STAGE = multiprocessing.cpu_count() +DATA_ARCHIVE_DIR = USER_RESOURCE_PATH +DIFF_VERSION_DEPTH = 2 # Logging section: LOG_FILE_PATH = os.path.join(MAIN_OUT_PATH, "generation.log") @@ -216,26 +215,20 @@ def init(default_settings_path: AnyStr): # Generator tool section: global USER_RESOURCE_PATH - global DATA_ARCHIVE_DIR - global DIFF_VERSION_DEPTH global NODE_STORAGE USER_RESOURCE_PATH = cfg.get_opt_path( "Generator tool", "USER_RESOURCE_PATH", USER_RESOURCE_PATH ) - DATA_ARCHIVE_DIR = cfg.get_opt_path( - "Generator tool", "DATA_ARCHIVE_DIR", DATA_ARCHIVE_DIR - ) - DIFF_VERSION_DEPTH = cfg.get_opt( - "Generator tool", "DIFF_VERSION_DEPTH", DIFF_VERSION_DEPTH - ) NODE_STORAGE = cfg.get_opt("Generator tool", "NODE_STORAGE", NODE_STORAGE) if not os.path.exists(USER_RESOURCE_PATH): from data_files import find_data_files + USER_RESOURCE_PATH = find_data_files("omim-data") assert USER_RESOURCE_PATH is not None import borders + # Issue: If maps_generator is installed in your system as a system # package and borders.init() is called first time, call borders.init() # might return False, because you need root permission. @@ -243,7 +236,26 @@ def init(default_settings_path: AnyStr): # Stages section: global NEED_PLANET_UPDATE + global DATA_ARCHIVE_DIR + global DIFF_VERSION_DEPTH + global THREADS_COUNT_FEATURES_STAGE NEED_PLANET_UPDATE = cfg.get_opt("Stages", "NEED_PLANET_UPDATE", NEED_PLANET_UPDATE) + DATA_ARCHIVE_DIR = cfg.get_opt_path( + "Generator tool", "DATA_ARCHIVE_DIR", DATA_ARCHIVE_DIR + ) + DIFF_VERSION_DEPTH = cfg.get_opt( + "Generator tool", "DIFF_VERSION_DEPTH", DIFF_VERSION_DEPTH + ) + + threads_count = int( + cfg.get_opt( + "Generator tool", + "THREADS_COUNT_FEATURES_STAGE", + THREADS_COUNT_FEATURES_STAGE, + ) + ) + if threads_count > 0: + THREADS_COUNT_FEATURES_STAGE = threads_count # Logging section: global LOG_FILE_PATH diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index 2338407e9c..570ef85f92 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -178,23 +178,27 @@ class StageMwm(Stage): @staticmethod def make_mwm(country: AnyStr, env: Env): - if country == WORLD_NAME: - StageIndex(country=country)(env) - StageCitiesIdsWorld(country=country)(env) - elif country == WORLD_COASTS_NAME: - StageIndex(country=country)(env) - else: - StageIndex(country=country)(env) - StageUgc(country=country)(env) - StagePopularity(country=country)(env) - StageSrtm(country=country)(env) - StageIsolinesInfo(country=country)(env) - StageDescriptions(country=country)(env) - StageRouting(country=country)(env) - StageRoutingTransit(country=country)(env) - StageMwmDiffs(country=country)(env) + world_stages = { + WORLD_NAME: [StageIndex, StageCitiesIdsWorld, StageMwmStatistics], + WORLD_COASTS_NAME: [StageIndex, StageMwmStatistics], + } + + mwm_stages = [ + StageIndex, + StageUgc, + StagePopularity, + StageSrtm, + StageIsolinesInfo, + StageDescriptions, + StageRouting, + StageRoutingTransit, + StageMwmDiffs, + StageMwmStatistics, + ] + + for stage in world_stages.get(country, mwm_stages): + stage(country=country)(env) - StageMwmStatistics(country=country)(env) env.finish_mwm(country) @@ -270,8 +274,10 @@ class StageRouting(Stage): @country_stage -@depends_from_internal(D(settings.SUBWAY_URL, PathProvider.subway_path), - D(settings.TRANSIT_URL, PathProvider.transit_path_experimental),) +@depends_from_internal( + D(settings.SUBWAY_URL, PathProvider.subway_path), + D(settings.TRANSIT_URL, PathProvider.transit_path_experimental), +) class StageRoutingTransit(Stage): def apply(self, env: Env, country, **kwargs): steps.step_routing_transit(env, country, **kwargs) diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 16fbf70049..4430fe15f7 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -129,7 +129,7 @@ def step_features(env: Env, **kwargs): dump_cities_boundaries=True, cities_boundaries_data=env.paths.cities_boundaries_path, generate_features=True, - threads_count=settings.THREADS_COUNT, + threads_count=settings.THREADS_COUNT_FEATURES_STAGE, **kwargs, ) diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 07481e8e29..ce7e25ea25 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -19,10 +19,6 @@ OMIM_PATH: ~/omim USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data # Do not change it. This is determined automatically. # NODE_STORAGE: map -# If you want to calculate diffs, you need to specify, where old maps are -DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH} -# You may specify, how many versions in the archive to use for diff calculation -DIFF_VERSION_DEPTH: 2 [Osm tools] @@ -35,6 +31,12 @@ OSM_TOOLS_PATH: ~/osmctools [Stages] # Run osmupdate tool for planet. NEED_PLANET_UPDATE: 0 +# Auto detection. +THREADS_COUNT_FEATURES_STAGE: 0 +# If you want to calculate diffs, you need to specify, where old maps are +DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH} +# You may specify, how many versions in the archive to use for diff calculation +DIFF_VERSION_DEPTH: 2 [Logging]