[python][generator] Added 'threads_count' option.

generator/generator_tool/generator_tool.cpp

@@ -197,6 +197,8 @@ DEFINE_bool(generate_traffic_keys, false,
 DEFINE_bool(dump_mwm_tmp, false, "Prints feature builder objects from .mwm.tmp");
 
 // Common.
+DEFINE_uint64(threads_count, 0, "Desired count of threads. If count equals zero, count of "
+                                "threads is set automatically.");
 DEFINE_bool(verbose, false, "Provide more detailed output.");
 
 using namespace generator;
@@ -212,7 +214,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv)
   google::ParseCommandLineFlags(&argc, &argv, true);
 
   Platform & pl = GetPlatform();
-  auto threadsCount = pl.CpuCores();
+  unsigned threadsCount = FLAGS_threads_count != 0 ? static_cast<unsigned>(FLAGS_threads_count)
+                                                   : pl.CpuCores();
 
   if (!FLAGS_user_resource_path.empty())
   {
@@ -385,7 +388,7 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv)
 
       /// @todo Make threads count according to environment (single mwm build or planet build).
       if (!indexer::BuildSearchIndexFromDataFile(path, country, true /* forceRebuild */,
-                                                 1 /* threadsCount */))
+                                                 threadsCount))
       {
         LOG(LCRITICAL, ("Error generating search index."));
       }

tools/python/maps_generator/README.md

@@ -100,6 +100,10 @@ SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json
 # UK_POSTCODES_URL:
 # US_POSTCODES_URL:
 
+[Common]
+# Auto detection.
+THREADS_COUNT: 0
+
 [Stats]
 # Path to rules for calculating statistics by type
 STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt

tools/python/maps_generator/generator/gen_tool.py

@@ -3,6 +3,7 @@ import logging
 import os
 import subprocess
 
+from . import settings
 from .exceptions import OptionNotFound
 from .exceptions import ValidationError
 from .exceptions import wait_and_raise_if_fail
@@ -45,6 +46,7 @@ class GenTool:
         "split_by_polygons": bool,
         "type_statistics": bool,
         "version": bool,
+        "threads_count": int,
         "booking_data": str,
         "promo_catalog_cities": str,
         "brands_data": str,
@@ -84,7 +86,7 @@ class GenTool:
         self.subprocess = None
         self.output = out
         self.error = err
-        self.options = {}
+        self.options = {"threads_count": 1}
         self.logger = logger
         self.add_options(**options)
 

tools/python/maps_generator/generator/settings.py

@@ -127,9 +127,6 @@ OSM_TOOLS_CC_FLAGS = [
     "-O3",
 ]
 
-# System:
-CPU_COUNT = multiprocessing.cpu_count()
-
 # Planet and coasts:
 PLANET_O5M = os.path.join(MAIN_OUT_PATH, PLANET + ".o5m")
 PLANET_PBF = os.path.join(MAIN_OUT_PATH, PLANET + ".osm.pbf")
@@ -140,6 +137,9 @@ if DEBUG:
     PLANET_URL = "http://osmz.ru/mwm/islands/islands.o5m"
     PLANET_MD5_URL = "https://cloud.mail.ru/public/5v2F/f7cSaEXBC"
 
+# Common:
+THREADS_COUNT = multiprocessing.cpu_count()
+
 # for lib logging
 LOGGING = {
     "version": 1,
@@ -261,6 +261,12 @@ def init(default_settings_path: AnyStr):
         "Stats", "STATS_TYPES_CONFIG", STATS_TYPES_CONFIG
     )
 
+    # Common:
+    global THREADS_COUNT
+    threads_count = int(cfg.get_opt("Common", "THREADS_COUNT", THREADS_COUNT))
+    if threads_count > 0:
+        THREADS_COUNT = threads_count
+
     # Planet and costs:
     global PLANET_O5M
     global PLANET_PBF

tools/python/maps_generator/generator/stages_declaration.py

@@ -194,7 +194,7 @@ class StageDownloadDescriptions(Stage):
 @mwm_stage
 class StageMwm(Stage):
     def apply(self, env: Env):
-        with ThreadPool() as pool:
+        with ThreadPool(settings.THREADS_COUNT) as pool:
             pool.map(
                 lambda c: StageMwm.make_mwm(c, env),
                 env.get_tmp_mwm_names(),
@@ -399,7 +399,7 @@ class StageStatistics(Stage):
 
         names = env.get_tmp_mwm_names()
         countries = filter(lambda x: x not in WORLDS_NAMES, names)
-        with ThreadPool() as pool:
+        with ThreadPool(settings.THREADS_COUNT) as pool:
             pool.map(partial(stage_mwm_statistics, env), countries)
 
         steps_info = get_stages_info(env.paths.log_path, {"statistics"})

tools/python/maps_generator/generator/steps.py

@@ -1,6 +1,7 @@
 """
 This file contains basic api for generator_tool and osm tools to generate maps.
 """
+import functools
 import logging
 import os
 import shutil
@@ -25,6 +26,16 @@ from ..utils.md5 import write_md5sum
 logger = logging.getLogger("maps_generator")
 
 
+def multithread_run_if_one_country(func):
+    @functools.wraps(func)
+    def wrap(env, *args, **kwargs):
+        if len(env.countries) == 1:
+            kwargs.update({"threads_count": settings.THREADS_COUNT})
+        func(env, *args, **kwargs)
+
+    return wrap
+
+
 def download_planet(planet: AnyStr):
     download_file(settings.PLANET_URL, planet)
     download_file(settings.PLANET_MD5_URL, md5(planet))
@@ -109,6 +120,7 @@ def step_features(env: Env, **kwargs):
         dump_cities_boundaries=True,
         cities_boundaries_data=env.paths.cities_boundaries_path,
         generate_features=True,
+        threads_count=settings.THREADS_COUNT,
         **kwargs,
     )
 
@@ -135,6 +147,7 @@ def run_gen_tool_with_recovery_country(env: Env, *args, **kwargs):
     kwargs["data_path"] = prev_data_path
 
 
+@multithread_run_if_one_country
 def _generate_common_index(env: Env, country: AnyStr, **kwargs):
     run_gen_tool(
         env.gen_tool,

tools/python/maps_generator/var/etc/map_generator.ini.default

@@ -40,6 +40,10 @@ SUBWAY_URL: http://osm-subway.maps.me/mapsme/latest.json
 # SRTM_PATH:
 # ISOLINES_PATH:
 
+[Common]
+# Auto detection.
+THREADS_COUNT: 0
+
 [Stats]
 STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt