From c754c32c5271c17af079ee35758369008c80794a Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 9 Oct 2020 12:38:47 +0300 Subject: [PATCH] [generator] Added 'cache_path' option. --- generator/generate_info.hpp | 7 +++++ generator/generator_tool/generator_tool.cpp | 7 ++++- generator/intermediate_data.cpp | 26 +++++++++---------- generator/osm_source.cpp | 4 +-- map/framework.cpp | 3 +-- tools/python/maps_generator/README.md | 6 +++-- tools/python/maps_generator/generator/env.py | 18 ++++++++++--- .../maps_generator/generator/gen_tool.py | 1 + .../maps_generator/generator/settings.py | 7 +++-- .../generator/stages_declaration.py | 1 + .../python/maps_generator/generator/steps.py | 10 +++++++ .../var/etc/map_generator.ini.default | 6 +++-- 12 files changed, 68 insertions(+), 28 deletions(-) diff --git a/generator/generate_info.hpp b/generator/generate_info.hpp index 507d78ebaf..41fbfc9559 100644 --- a/generator/generate_info.hpp +++ b/generator/generate_info.hpp @@ -37,6 +37,8 @@ struct GenerateInfo // Directory for all intermediate files. std::string m_intermediateDir; + std::string m_cacheDir; + // Directory with isolines files. std::string m_isolinesDir; @@ -115,5 +117,10 @@ struct GenerateInfo { return base::JoinPath(m_intermediateDir, fileName + ext); } + + std::string GetCacheFileName(std::string const & fileName, std::string const & ext = "") const + { + return base::JoinPath(m_cacheDir, fileName + ext); + } }; } // namespace feature diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 4fcf99dda0..7b14a2b00e 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -100,7 +100,10 @@ DEFINE_string(osm_file_name, "", "Input osm area file."); DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]."); DEFINE_string(data_path, "", GetDataPathHelp()); DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc."); -DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations."); +DEFINE_string(intermediate_data_path, "", "Path to stored intermediate data."); +DEFINE_string(cache_path, "", + "Path to stored caches for nodes, ways, relations. " + "If 'cache_path' is empty, caches are stored to 'intermediate_data_path'."); DEFINE_string(output, "", "File name for process (without 'mwm' ext)."); DEFINE_bool(preload_cache, false, "Preload all ways and relations cache."); DEFINE_string(node_storage, "map", @@ -241,6 +244,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) genInfo.m_intermediateDir = FLAGS_intermediate_data_path.empty() ? path : base::AddSlashIfNeeded(FLAGS_intermediate_data_path); + genInfo.m_cacheDir = FLAGS_cache_path.empty() ? genInfo.m_intermediateDir + : base::AddSlashIfNeeded(FLAGS_cache_path); genInfo.m_targetDir = genInfo.m_tmpDir = path; /// @todo Probably, it's better to add separate option for .mwm.tmp files. diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index 2b746fbf75..9e3717a51c 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -376,25 +376,25 @@ IndexFileReader const & IntermediateDataObjectsCache::AllocatedObjects::GetOrCre // IntermediateDataReader IntermediateDataReader::IntermediateDataReader( - IntermediateDataObjectsCache::AllocatedObjects & objs, - feature::GenerateInfo const & info) + IntermediateDataObjectsCache::AllocatedObjects & objs, feature::GenerateInfo const & info) : m_nodes(objs.GetPointStorageReader()) - , m_ways(objs, info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache) - , m_relations(objs, info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache) - , m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))) - , m_wayToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))) - , m_relationToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT))) + , m_ways(objs, info.GetCacheFileName(WAYS_FILE), info.m_preloadCache) + , m_relations(objs, info.GetCacheFileName(RELATIONS_FILE), info.m_preloadCache) + , m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetCacheFileName(NODES_FILE, ID2REL_EXT))) + , m_wayToRelations(objs.GetOrCreateIndexReader(info.GetCacheFileName(WAYS_FILE, ID2REL_EXT))) + , m_relationToRelations( + objs.GetOrCreateIndexReader(info.GetCacheFileName(RELATIONS_FILE, ID2REL_EXT))) {} // IntermediateDataWriter IntermediateDataWriter::IntermediateDataWriter(PointStorageWriterInterface & nodes, feature::GenerateInfo const & info) : m_nodes(nodes) - , m_ways(info.GetIntermediateFileName(WAYS_FILE)) - , m_relations(info.GetIntermediateFileName(RELATIONS_FILE)) - , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) - , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) - , m_relationToRelations(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT)) + , m_ways(info.GetCacheFileName(WAYS_FILE)) + , m_relations(info.GetCacheFileName(RELATIONS_FILE)) + , m_nodeToRelations(info.GetCacheFileName(NODES_FILE, ID2REL_EXT)) + , m_wayToRelations(info.GetCacheFileName(WAYS_FILE, ID2REL_EXT)) + , m_relationToRelations(info.GetCacheFileName(RELATIONS_FILE, ID2REL_EXT)) {} void IntermediateDataWriter::AddRelation(Key id, RelationElement const & e) @@ -458,7 +458,7 @@ IntermediateData::IntermediateData(IntermediateDataObjectsCache & objectsCache, , m_info(info) { auto & allocatedObjects = m_objectsCache.GetOrCreatePointStorageReader( - info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE)); + info.m_nodeStorageType, info.GetCacheFileName(NODES_FILE)); m_reader = make_shared(allocatedObjects, info); } diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 360f24da4a..6a76031595 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -246,8 +246,8 @@ bool ProcessorOsmElementsFromXml::TryRead(OsmElement & element) bool GenerateIntermediateData(feature::GenerateInfo & info) { - auto nodes = cache::CreatePointStorageWriter(info.m_nodeStorageType, - info.GetIntermediateFileName(NODES_FILE)); + auto nodes = + cache::CreatePointStorageWriter(info.m_nodeStorageType, info.GetCacheFileName(NODES_FILE)); cache::IntermediateDataWriter cache(*nodes, info); TownsDumper towns; SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); diff --git a/map/framework.cpp b/map/framework.cpp index ca16a4624f..b26a28240c 100644 --- a/map/framework.cpp +++ b/map/framework.cpp @@ -796,8 +796,7 @@ kml::MarkGroupId Framework::AddCategory(string const & categoryName) void Framework::FillPointInfoForBookmark(Bookmark const & bmk, place_page::Info & info) const { auto types = feature::TypesHolder::FromTypesIndexes(bmk.GetData().m_featureTypes); - FillPointInfo(info, bmk.GetPivot(), {} /* customTitle */, [&types](FeatureType & ft) - { + FillPointInfo(info, bmk.GetPivot(), {} /* customTitle */, [&types](FeatureType & ft) { return !types.Empty() && feature::TypesHolder(ft).Equals(types); }); } diff --git a/tools/python/maps_generator/README.md b/tools/python/maps_generator/README.md index 19b07c9318..3de4d475ed 100644 --- a/tools/python/maps_generator/README.md +++ b/tools/python/maps_generator/README.md @@ -42,10 +42,12 @@ maps_generator$ vim var/etc/map_generator.ini ```ini [Main] -# The path where the planet will be downloaded and the maps are generated. -MAIN_OUT_PATH: ~/maps_build # If the flag DEBUG is set a special small planet file will be downloaded. DEBUG: 1 +# The path where the planet will be downloaded and the maps are generated. +MAIN_OUT_PATH: ~/maps_build +# The path where caches for nodes, ways, relations are stored. +# CACHE_PATH: [Developer] diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 0a874a8fdf..3f6aac6eb3 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -45,7 +45,7 @@ def get_all_countries_list(borders_path: AnyStr) -> List[AnyStr]: def create_if_not_exist_path(path: AnyStr) -> bool: """Creates directory if it doesn't exist.""" try: - os.mkdir(path) + os.makedirs(path) logger.info(f"Create {path} ...") return True except FileExistsError: @@ -118,8 +118,9 @@ class PathProvider: PathProvider is used for building paths for a maps generation. """ - def __init__(self, build_path: AnyStr, mwm_version: AnyStr): + def __init__(self, build_path: AnyStr, build_name:AnyStr, mwm_version: AnyStr): self.build_path = build_path + self.build_name = build_name self.mwm_version = mwm_version create_if_not_exist_path(self.build_path) @@ -134,10 +135,19 @@ class PathProvider: """ return os.path.join(self.build_path, "intermediate_data") + @property + @create_if_not_exist + def cache_path(self) -> AnyStr: + """cache_path contains caches for nodes, ways, relations.""" + if not settings.CACHE_PATH: + return self.intermediate_data_path + + return os.path.join(settings.CACHE_PATH, self.build_name) + @property @create_if_not_exist def data_path(self) -> AnyStr: - """It's a synonum for intermediate_data_path.""" + """It's a synonym for intermediate_data_path.""" return self.intermediate_data_path @property @@ -405,7 +415,7 @@ class Env: logger.info(f"Build name is {self.build_name}.") logger.info(f"Build path is {self.build_path}.") - self.paths = PathProvider(self.build_path, self.mwm_version) + self.paths = PathProvider(self.build_path, self.build_name, self.mwm_version) Version.write(self.build_path, self.planet_version) self.setup_borders() diff --git a/tools/python/maps_generator/generator/gen_tool.py b/tools/python/maps_generator/generator/gen_tool.py index 491377e620..1915a3e4a4 100644 --- a/tools/python/maps_generator/generator/gen_tool.py +++ b/tools/python/maps_generator/generator/gen_tool.py @@ -51,6 +51,7 @@ class GenTool: "promo_catalog_cities": str, "brands_data": str, "brands_translations_data": str, + "cache_path": str, "cities_boundaries_data": str, "data_path": str, "dump_wikipedia_urls": str, diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index eee97a09d1..cea43c67d5 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -80,6 +80,7 @@ _HOME_PATH = str(Path.home()) _WORK_PATH = _HOME_PATH TMPDIR = os.path.join(_HOME_PATH, "tmp") MAIN_OUT_PATH = os.path.join(_WORK_PATH, "generation") +CACHE_PATH = "" # Developer section: BUILD_PATH = os.path.join(_WORK_PATH, "omim-build-release") @@ -186,12 +187,14 @@ def init(default_settings_path: AnyStr): # Main section: global DEBUG - global MAIN_OUT_PATH global TMPDIR + global MAIN_OUT_PATH + global CACHE_PATH _DEBUG = cfg.get_opt("Main", "DEBUG") DEBUG = DEBUG if _DEBUG is None else int(_DEBUG) - MAIN_OUT_PATH = cfg.get_opt_path("Main", "MAIN_OUT_PATH", MAIN_OUT_PATH) TMPDIR = cfg.get_opt_path("Main", "TMPDIR", TMPDIR) + MAIN_OUT_PATH = cfg.get_opt_path("Main", "MAIN_OUT_PATH", MAIN_OUT_PATH) + CACHE_PATH = cfg.get_opt_path("Main", "CACHE_PATH", CACHE_PATH) # Developer section: global BUILD_PATH diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index 5c50637caa..6908bf46e5 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -147,6 +147,7 @@ class StageDownloadDescriptions(Stage): out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, dump_wikipedia_urls=env.paths.wiki_url_path, idToWikidata=env.paths.id_to_wikidata_path, diff --git a/tools/python/maps_generator/generator/steps.py b/tools/python/maps_generator/generator/steps.py index 75f314b1b8..16fbf70049 100644 --- a/tools/python/maps_generator/generator/steps.py +++ b/tools/python/maps_generator/generator/steps.py @@ -95,6 +95,7 @@ def step_preprocess(env: Env, **kwargs): out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, osm_file_type="o5m", osm_file_name=env.paths.planet_o5m, node_storage=env.node_storage, @@ -120,6 +121,7 @@ def step_features(env: Env, **kwargs): err=env.get_subprocess_out(), data_path=env.paths.data_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, osm_file_type="o5m", osm_file_name=env.paths.planet_o5m, node_storage=env.node_storage, @@ -162,6 +164,7 @@ def _generate_common_index(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, node_storage=env.node_storage, planet_version=env.planet_version, @@ -213,6 +216,7 @@ def step_ugc(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, ugc_data=env.paths.ugc_path, output=country, @@ -228,6 +232,7 @@ def step_popularity(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, popular_places_data=env.paths.popularity_path, generate_popular_places=True, @@ -244,6 +249,7 @@ def step_srtm(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, srtm_path=env.paths.srtm_path(), output=country, @@ -259,6 +265,7 @@ def step_isolines_info(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, generate_isolines_info=True, isolines_path=PathProvider.isolines_path(), @@ -290,6 +297,7 @@ def step_routing(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, cities_boundaries_data=env.paths.cities_boundaries_path, generate_maxspeed=True, @@ -312,6 +320,7 @@ def step_routing_transit(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, transit_path=env.paths.transit_path, transit_path_experimental=env.paths.transit_path_experimental, @@ -330,6 +339,7 @@ def step_statistics(env: Env, country: AnyStr, **kwargs): err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, + cache_path=env.paths.cache_path, user_resource_path=env.paths.user_resource_path, type_statistics=True, output=country, diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 1b3918a256..978668e7cd 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -1,8 +1,10 @@ [Main] -# The path where the planet will be downloaded and the maps are generated. -MAIN_OUT_PATH: ~/maps_build # If the flag DEBUG is set a special small planet file will be downloaded. DEBUG: 1 +# The path where the planet will be downloaded and the maps are generated. +MAIN_OUT_PATH: ~/maps_build +# The path where caches for nodes, ways, relations are stored. +# CACHE_PATH: [Developer]