kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

[python] Fixed data race.

Browse source
This commit is contained in:
Maksim Andrianov 2019-05-22 22:40:21 +03:00 • committed by mpimenov
parent 207871a55d
commit 28a21c83ee
6 changed files with 46 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
tools/python/maps_generator/generator/basic_stages.py
View file

@ -26,13 +26,13 @@ def convert_planet(tool, in_planet, out_planet, output=subprocess.DEVNULL,
def stage_download_and_convert_planet(env, **kwargs):
if not is_verified(settings.PLANET_PBF):
download_planet(settings.PLANET_PBF, output=env.subprocess_out,
error=env.subprocess_out)
download_planet(settings.PLANET_PBF, output=env.get_subprocess_out(),
error=env.get_subprocess_out())
convert_planet(env[settings.OSM_TOOL_CONVERT],
settings.PLANET_PBF, settings.PLANET_O5M,
output=env.subprocess_out,
error=env.subprocess_out)
output=env.get_subprocess_out(),
error=env.get_subprocess_out())
os.remove(settings.PLANET_PBF)
os.remove(md5(settings.PLANET_PBF))
@ -40,8 +40,8 @@ def stage_download_and_convert_planet(env, **kwargs):
def stage_update_planet(env, **kwargs):
tmp = settings.PLANET_O5M + ".tmp"
osmupdate(env[settings.OSM_TOOL_UPDATE], settings.PLANET_O5M, tmp,
output=env.subprocess_out,
error=env.subprocess_out,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
**kwargs)
os.remove(settings.PLANET_O5M)
os.rename(tmp, settings.PLANET_O5M)
@ -50,8 +50,8 @@ def stage_update_planet(env, **kwargs):
def stage_preprocess(env, **kwargs):
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.intermediate_path,
osm_file_type="o5m",
osm_file_name=settings.PLANET_O5M,

12
tools/python/maps_generator/generator/coastline.py
View file

@ -16,12 +16,12 @@ def filter_coastline(name_executable, in_file, out_file,
def make_coastline(env):
coastline_o5m = os.path.join(env.coastline_path, "coastline.o5m")
filter_coastline(env[settings.OSM_TOOL_FILTER], settings.PLANET_O5M,
coastline_o5m, output=env.subprocess_out,
error=env.subprocess_out)
coastline_o5m, output=env.get_subprocess_out(),
error=env.get_subprocess_out())
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.coastline_path,
osm_file_type="o5m",
osm_file_name=coastline_o5m,
@ -29,8 +29,8 @@ def make_coastline(env):
preprocess=True)
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.coastline_path,
osm_file_type="o5m",
osm_file_name=coastline_o5m,

2
tools/python/maps_generator/generator/decorators.py
View file

@ -76,7 +76,7 @@ def country_stage_log(func):
_logger.info(f"{stage_formatted}: start ...")
t = time.time()
with open(log_file, "a+") as l:
env.set_subprocess_out(l)
env.set_subprocess_out(l, country)
func(env, country, *args, logger=_logger, **kwargs)
d = time.time() - t
_logger.info(f"{stage_formatted}: finished in "

16
tools/python/maps_generator/generator/env.py
View file

@ -113,7 +113,8 @@ class Env:
self.coastline_tmp_path = os.path.join(self.coastline_path, "tmp")
self._create_if_not_exist(self.coastline_tmp_path)
self.subprocess_out = None
self._subprocess_out = None
self._subprocess_countries_out = {}
self.descriptions_path = os.path.join(self.intermediate_path,
"descriptions")
@ -206,8 +207,17 @@ class Env:
def finish_mwm(self, mwm_name):
self.countries_meta[mwm_name]["status"].finish()
def set_subprocess_out(self, subprocess_out):
self.subprocess_out = subprocess_out
def set_subprocess_out(self, subprocess_out, country=None):
if country is None:
self._subprocess_out = subprocess_out
else:
self._subprocess_countries_out[country] = subprocess_out
def get_subprocess_out(self, country=None):
if country is None:
return self._subprocess_out
else:
return self._subprocess_countries_out[country]
@staticmethod
def _logging_setup():

24
tools/python/maps_generator/generator/maps_stages.py
View file

@ -28,8 +28,8 @@ def run_gen_tool_with_recovery_country(env, *args, **kwargs):
def stage_index_world(env, country, **kwargs):
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
@ -50,8 +50,8 @@ def stage_index(env, country, **kwargs):
def stage_coastline_index(env, country, **kwargs):
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
@ -67,8 +67,8 @@ def stage_ugc(env, country, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
@ -82,8 +82,8 @@ def stage_popularity(env, country, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
@ -98,8 +98,8 @@ def stage_routing(env, country, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
@ -117,8 +117,8 @@ def stage_routing_transit(env, country, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,

12
tools/python/maps_generator/maps_generator.py
View file

@ -85,8 +85,8 @@ def stage_features(env):
run_gen_tool(
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
data_path=env.data_path,
intermediate_data_path=env.intermediate_path,
osm_file_type="o5m",
@ -182,8 +182,8 @@ def stage_mwm(env):
@stage
def stage_descriptions(env):
run_gen_tool(env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.intermediate_path,
user_resource_path=env.user_resource_path,
dump_wikipedia_urls=env.wiki_url_path,
@ -201,8 +201,8 @@ def stage_descriptions(env):
maps_stages.run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.subprocess_out,
err=env.subprocess_out,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.mwm_path,
user_resource_path=env.user_resource_path,
wikipedia_pages=env.descriptions_path,