RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] Added 'cache_path' option.

Browse source
This commit is contained in:
Maksim Andrianov 2020-10-09 12:38:47 +03:00 • committed by mpimenov
parent 519fe79237
commit c754c32c52
12 changed files with 68 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
generator/generate_info.hpp
View file

@ -37,6 +37,8 @@ struct GenerateInfo
// Directory for all intermediate files.
std::string m_intermediateDir;
std::string m_cacheDir;
// Directory with isolines files.
std::string m_isolinesDir;
@ -115,5 +117,10 @@ struct GenerateInfo
{
return base::JoinPath(m_intermediateDir, fileName + ext);
}
std::string GetCacheFileName(std::string const & fileName, std::string const & ext = "") const
{
return base::JoinPath(m_cacheDir, fileName + ext);
}
};
} // namespace feature

7
generator/generator_tool/generator_tool.cpp
View file

@ -100,7 +100,10 @@ DEFINE_string(osm_file_name, "", "Input osm area file.");
DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m].");
DEFINE_string(data_path, "", GetDataPathHelp());
DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc.");
DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations.");
DEFINE_string(intermediate_data_path, "", "Path to stored intermediate data.");
DEFINE_string(cache_path, "",
"Path to stored caches for nodes, ways, relations. "
"If 'cache_path' is empty, caches are stored to 'intermediate_data_path'.");
DEFINE_string(output, "", "File name for process (without 'mwm' ext).");
DEFINE_bool(preload_cache, false, "Preload all ways and relations cache.");
DEFINE_string(node_storage, "map",
@ -241,6 +244,8 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv)
genInfo.m_intermediateDir = FLAGS_intermediate_data_path.empty()
? path
: base::AddSlashIfNeeded(FLAGS_intermediate_data_path);
genInfo.m_cacheDir = FLAGS_cache_path.empty() ? genInfo.m_intermediateDir
: base::AddSlashIfNeeded(FLAGS_cache_path);
genInfo.m_targetDir = genInfo.m_tmpDir = path;
/// @todo Probably, it's better to add separate option for .mwm.tmp files.

26
generator/intermediate_data.cpp
View file

@ -376,25 +376,25 @@ IndexFileReader const & IntermediateDataObjectsCache::AllocatedObjects::GetOrCre
// IntermediateDataReader
IntermediateDataReader::IntermediateDataReader(
IntermediateDataObjectsCache::AllocatedObjects & objs,
feature::GenerateInfo const & info)
IntermediateDataObjectsCache::AllocatedObjects & objs, feature::GenerateInfo const & info)
: m_nodes(objs.GetPointStorageReader())
, m_ways(objs, info.GetIntermediateFileName(WAYS_FILE), info.m_preloadCache)
, m_relations(objs, info.GetIntermediateFileName(RELATIONS_FILE), info.m_preloadCache)
, m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)))
, m_wayToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)))
, m_relationToRelations(objs.GetOrCreateIndexReader(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT)))
, m_ways(objs, info.GetCacheFileName(WAYS_FILE), info.m_preloadCache)
, m_relations(objs, info.GetCacheFileName(RELATIONS_FILE), info.m_preloadCache)
, m_nodeToRelations(objs.GetOrCreateIndexReader(info.GetCacheFileName(NODES_FILE, ID2REL_EXT)))
, m_wayToRelations(objs.GetOrCreateIndexReader(info.GetCacheFileName(WAYS_FILE, ID2REL_EXT)))
, m_relationToRelations(
objs.GetOrCreateIndexReader(info.GetCacheFileName(RELATIONS_FILE, ID2REL_EXT)))
{}
// IntermediateDataWriter
IntermediateDataWriter::IntermediateDataWriter(PointStorageWriterInterface & nodes,
feature::GenerateInfo const & info)
: m_nodes(nodes)
, m_ways(info.GetIntermediateFileName(WAYS_FILE))
, m_relations(info.GetIntermediateFileName(RELATIONS_FILE))
, m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT))
, m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT))
, m_relationToRelations(info.GetIntermediateFileName(RELATIONS_FILE, ID2REL_EXT))
, m_ways(info.GetCacheFileName(WAYS_FILE))
, m_relations(info.GetCacheFileName(RELATIONS_FILE))
, m_nodeToRelations(info.GetCacheFileName(NODES_FILE, ID2REL_EXT))
, m_wayToRelations(info.GetCacheFileName(WAYS_FILE, ID2REL_EXT))
, m_relationToRelations(info.GetCacheFileName(RELATIONS_FILE, ID2REL_EXT))
{}
void IntermediateDataWriter::AddRelation(Key id, RelationElement const & e)
@ -458,7 +458,7 @@ IntermediateData::IntermediateData(IntermediateDataObjectsCache & objectsCache,
, m_info(info)
{
auto & allocatedObjects = m_objectsCache.GetOrCreatePointStorageReader(
info.m_nodeStorageType, info.GetIntermediateFileName(NODES_FILE));
info.m_nodeStorageType, info.GetCacheFileName(NODES_FILE));
m_reader = make_shared<IntermediateDataReader>(allocatedObjects, info);
}

4
generator/osm_source.cpp
View file

@ -246,8 +246,8 @@ bool ProcessorOsmElementsFromXml::TryRead(OsmElement & element)
bool GenerateIntermediateData(feature::GenerateInfo & info)
{
auto nodes = cache::CreatePointStorageWriter(info.m_nodeStorageType,
info.GetIntermediateFileName(NODES_FILE));
auto nodes =
cache::CreatePointStorageWriter(info.m_nodeStorageType, info.GetCacheFileName(NODES_FILE));
cache::IntermediateDataWriter cache(*nodes, info);
TownsDumper towns;
SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName);

3
map/framework.cpp
View file

@ -796,8 +796,7 @@ kml::MarkGroupId Framework::AddCategory(string const & categoryName)
void Framework::FillPointInfoForBookmark(Bookmark const & bmk, place_page::Info & info) const
{
auto types = feature::TypesHolder::FromTypesIndexes(bmk.GetData().m_featureTypes);
FillPointInfo(info, bmk.GetPivot(), {} /* customTitle */, [&types](FeatureType & ft)
{
FillPointInfo(info, bmk.GetPivot(), {} /* customTitle */, [&types](FeatureType & ft) {
return !types.Empty() && feature::TypesHolder(ft).Equals(types);
});
}

6
tools/python/maps_generator/README.md
View file

@ -42,10 +42,12 @@ maps_generator$ vim var/etc/map_generator.ini
```ini
[Main]
# The path where the planet will be downloaded and the maps are generated.
MAIN_OUT_PATH: ~/maps_build
# If the flag DEBUG is set a special small planet file will be downloaded.
DEBUG: 1
# The path where the planet will be downloaded and the maps are generated.
MAIN_OUT_PATH: ~/maps_build
# The path where caches for nodes, ways, relations are stored.
# CACHE_PATH:
[Developer]

18
tools/python/maps_generator/generator/env.py
View file

@ -45,7 +45,7 @@ def get_all_countries_list(borders_path: AnyStr) -> List[AnyStr]:
def create_if_not_exist_path(path: AnyStr) -> bool:
"""Creates directory if it doesn't exist."""
try:
os.mkdir(path)
os.makedirs(path)
logger.info(f"Create {path} ...")
return True
except FileExistsError:
@ -118,8 +118,9 @@ class PathProvider:
PathProvider is used for building paths for a maps generation.
"""
def __init__(self, build_path: AnyStr, mwm_version: AnyStr):
def __init__(self, build_path: AnyStr, build_name:AnyStr, mwm_version: AnyStr):
self.build_path = build_path
self.build_name = build_name
self.mwm_version = mwm_version
create_if_not_exist_path(self.build_path)
@ -134,10 +135,19 @@ class PathProvider:
"""
return os.path.join(self.build_path, "intermediate_data")
@property
@create_if_not_exist
def cache_path(self) -> AnyStr:
"""cache_path contains caches for nodes, ways, relations."""
if not settings.CACHE_PATH:
return self.intermediate_data_path
return os.path.join(settings.CACHE_PATH, self.build_name)
@property
@create_if_not_exist
def data_path(self) -> AnyStr:
"""It's a synonum for intermediate_data_path."""
"""It's a synonym for intermediate_data_path."""
return self.intermediate_data_path
@property
@ -405,7 +415,7 @@ class Env:
logger.info(f"Build name is {self.build_name}.")
logger.info(f"Build path is {self.build_path}.")
self.paths = PathProvider(self.build_path, self.mwm_version)
self.paths = PathProvider(self.build_path, self.build_name, self.mwm_version)
Version.write(self.build_path, self.planet_version)
self.setup_borders()

1
tools/python/maps_generator/generator/gen_tool.py
View file

@ -51,6 +51,7 @@ class GenTool:
"promo_catalog_cities": str,
"brands_data": str,
"brands_translations_data": str,
"cache_path": str,
"cities_boundaries_data": str,
"data_path": str,
"dump_wikipedia_urls": str,

7
tools/python/maps_generator/generator/settings.py
View file

@ -80,6 +80,7 @@ _HOME_PATH = str(Path.home())
_WORK_PATH = _HOME_PATH
TMPDIR = os.path.join(_HOME_PATH, "tmp")
MAIN_OUT_PATH = os.path.join(_WORK_PATH, "generation")
CACHE_PATH = ""
# Developer section:
BUILD_PATH = os.path.join(_WORK_PATH, "omim-build-release")
@ -186,12 +187,14 @@ def init(default_settings_path: AnyStr):
# Main section:
global DEBUG
global MAIN_OUT_PATH
global TMPDIR
global MAIN_OUT_PATH
global CACHE_PATH
_DEBUG = cfg.get_opt("Main", "DEBUG")
DEBUG = DEBUG if _DEBUG is None else int(_DEBUG)
MAIN_OUT_PATH = cfg.get_opt_path("Main", "MAIN_OUT_PATH", MAIN_OUT_PATH)
TMPDIR = cfg.get_opt_path("Main", "TMPDIR", TMPDIR)
MAIN_OUT_PATH = cfg.get_opt_path("Main", "MAIN_OUT_PATH", MAIN_OUT_PATH)
CACHE_PATH = cfg.get_opt_path("Main", "CACHE_PATH", CACHE_PATH)
# Developer section:
global BUILD_PATH

1
tools/python/maps_generator/generator/stages_declaration.py
View file

@ -147,6 +147,7 @@ class StageDownloadDescriptions(Stage):
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
dump_wikipedia_urls=env.paths.wiki_url_path,
idToWikidata=env.paths.id_to_wikidata_path,

10
tools/python/maps_generator/generator/steps.py
View file

@ -95,6 +95,7 @@ def step_preprocess(env: Env, **kwargs):
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
@ -120,6 +121,7 @@ def step_features(env: Env, **kwargs):
err=env.get_subprocess_out(),
data_path=env.paths.data_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
@ -162,6 +164,7 @@ def _generate_common_index(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
node_storage=env.node_storage,
planet_version=env.planet_version,
@ -213,6 +216,7 @@ def step_ugc(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
ugc_data=env.paths.ugc_path,
output=country,
@ -228,6 +232,7 @@ def step_popularity(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
popular_places_data=env.paths.popularity_path,
generate_popular_places=True,
@ -244,6 +249,7 @@ def step_srtm(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
srtm_path=env.paths.srtm_path(),
output=country,
@ -259,6 +265,7 @@ def step_isolines_info(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
generate_isolines_info=True,
isolines_path=PathProvider.isolines_path(),
@ -290,6 +297,7 @@ def step_routing(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
cities_boundaries_data=env.paths.cities_boundaries_path,
generate_maxspeed=True,
@ -312,6 +320,7 @@ def step_routing_transit(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
transit_path=env.paths.transit_path,
transit_path_experimental=env.paths.transit_path_experimental,
@ -330,6 +339,7 @@ def step_statistics(env: Env, country: AnyStr, **kwargs):
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
type_statistics=True,
output=country,

6
tools/python/maps_generator/var/etc/map_generator.ini.default
View file

@ -1,8 +1,10 @@
[Main]
# The path where the planet will be downloaded and the maps are generated.
MAIN_OUT_PATH: ~/maps_build
# If the flag DEBUG is set a special small planet file will be downloaded.
DEBUG: 1
# The path where the planet will be downloaded and the maps are generated.
MAIN_OUT_PATH: ~/maps_build
# The path where caches for nodes, ways, relations are stored.
# CACHE_PATH:
[Developer]