diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 6f0907f1b9..cc8555b03a 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -197,6 +197,8 @@ DEFINE_bool(generate_traffic_keys, false, DEFINE_bool(dump_mwm_tmp, false, "Prints feature builder objects from .mwm.tmp"); // Common. +DEFINE_uint64(threads_count, 0, "Desired count of threads. If count equals zero, count of " + "threads is set automatically."); DEFINE_bool(verbose, false, "Provide more detailed output."); using namespace generator; @@ -212,7 +214,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) google::ParseCommandLineFlags(&argc, &argv, true); Platform & pl = GetPlatform(); - auto threadsCount = pl.CpuCores(); + unsigned threadsCount = FLAGS_threads_count != 0 ? static_cast(FLAGS_threads_count) + : pl.CpuCores(); if (!FLAGS_user_resource_path.empty()) { @@ -385,7 +388,7 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) /// @todo Make threads count according to environment (single mwm build or planet build). if (!indexer::BuildSearchIndexFromDataFile(path, country, true /* forceRebuild */, - 1 /* threadsCount */)) + threadsCount)) { LOG(LCRITICAL, ("Error generating search index.")); } diff --git a/tools/python/maps_generator/README.md b/tools/python/maps_generator/README.md index b11ff6abe0..de98c61ecc 100644 --- a/tools/python/maps_generator/README.md +++ b/tools/python/maps_generator/README.md @@ -100,6 +100,10 @@ SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json # UK_POSTCODES_URL: # US_POSTCODES_URL: +[Common] +# Auto detection. +THREADS_COUNT: 0 + [Stats] # Path to rules for calculating statistics by type STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt diff --git a/tools/python/maps_generator/generator/gen_tool.py b/tools/python/maps_generator/generator/gen_tool.py index 9c365ed61d..b35bfab546 100644 --- a/tools/python/maps_generator/generator/gen_tool.py +++ b/tools/python/maps_generator/generator/gen_tool.py @@ -3,6 +3,7 @@ import logging import os import subprocess +from . import settings from .exceptions import OptionNotFound from .exceptions import ValidationError from .exceptions import wait_and_raise_if_fail @@ -45,6 +46,7 @@ class GenTool: "split_by_polygons": bool, "type_statistics": bool, "version": bool, + "threads_count": int, "booking_data": str, "promo_catalog_cities": str, "brands_data": str, @@ -84,7 +86,7 @@ class GenTool: self.subprocess = None self.output = out self.error = err - self.options = {} + self.options = {"threads_count": 1} self.logger = logger self.add_options(**options) diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index 490bc6af57..bc100c59c8 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -127,9 +127,6 @@ OSM_TOOLS_CC_FLAGS = [ "-O3", ] -# System: -CPU_COUNT = multiprocessing.cpu_count() - # Planet and coasts: PLANET_O5M = os.path.join(MAIN_OUT_PATH, PLANET + ".o5m") PLANET_PBF = os.path.join(MAIN_OUT_PATH, PLANET + ".osm.pbf") @@ -140,6 +137,9 @@ if DEBUG: PLANET_URL = "http://osmz.ru/mwm/islands/islands.o5m" PLANET_MD5_URL = "https://cloud.mail.ru/public/5v2F/f7cSaEXBC" +# Common: +THREADS_COUNT = multiprocessing.cpu_count() + # for lib logging LOGGING = { "version": 1, @@ -261,6 +261,12 @@ def init(default_settings_path: AnyStr): "Stats", "STATS_TYPES_CONFIG", STATS_TYPES_CONFIG ) + # Common: + global THREADS_COUNT + threads_count = int(cfg.get_opt("Common", "THREADS_COUNT", THREADS_COUNT)) + if threads_count > 0: + THREADS_COUNT = threads_count + # Planet and costs: global PLANET_O5M global PLANET_PBF diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index d6e634a90d..02d16c5e57 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -194,7 +194,7 @@ class StageDownloadDescriptions(Stage): @mwm_stage class StageMwm(Stage): def apply(self, env: Env): - with ThreadPool() as pool: + with ThreadPool(settings.THREADS_COUNT) as pool: pool.map( lambda c: StageMwm.make_mwm(c, env), env.get_tmp_mwm_names(), @@ -399,7 +399,7 @@ class StageStatistics(Stage): names = env.get_tmp_mwm_names() countries = filter(lambda x: x not in WORLDS_NAMES, names) - with ThreadPool() as pool: + with ThreadPool(settings.THREADS_COUNT) as pool: pool.map(partial(stage_mwm_statistics, env), countries) steps_info = get_stages_info(env.paths.log_path, {"statistics"}) diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 42ab5d833e..ab9fa657e0 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -1,6 +1,7 @@ """ This file contains basic api for generator_tool and osm tools to generate maps. """ +import functools import logging import os import shutil @@ -25,6 +26,16 @@ from ..utils.md5 import write_md5sum logger = logging.getLogger("maps_generator") +def multithread_run_if_one_country(func): + @functools.wraps(func) + def wrap(env, *args, **kwargs): + if len(env.countries) == 1: + kwargs.update({"threads_count": settings.THREADS_COUNT}) + func(env, *args, **kwargs) + + return wrap + + def download_planet(planet: AnyStr): download_file(settings.PLANET_URL, planet) download_file(settings.PLANET_MD5_URL, md5(planet)) @@ -109,6 +120,7 @@ def step_features(env: Env, **kwargs): dump_cities_boundaries=True, cities_boundaries_data=env.paths.cities_boundaries_path, generate_features=True, + threads_count=settings.THREADS_COUNT, **kwargs, ) @@ -135,6 +147,7 @@ def run_gen_tool_with_recovery_country(env: Env, *args, **kwargs): kwargs["data_path"] = prev_data_path +@multithread_run_if_one_country def _generate_common_index(env: Env, country: AnyStr, **kwargs): run_gen_tool( env.gen_tool, diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 9f859c7c3a..27ad1cdf03 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -40,6 +40,10 @@ SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json # SRTM_PATH: # ISOLINES_PATH: +[Common] +# Auto detection. +THREADS_COUNT: 0 + [Stats] STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt