gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[python] Supported download coasts files.

Browse source
This commit is contained in:
Maksim Andrianov 2019-05-21 16:27:21 +03:00 • committed by gmoryes
parent e78943ca12
commit ca4426daef
6 changed files with 46 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
tools/python/maps_generator/README.md
View file

@ -63,7 +63,8 @@ OSM_TOOLS_PATH: ~/osmctools
# PLANET_URL:
# The url to the file with md5 sum of the planet.
# PLANET_MD5_URL:
# The url to WorldCoasts.geom and WorldCoasts.rawgeom(without file name).
# PLANET_COASTS_URL:
# The url to the subway file.
# SUBWAY_URL:
@ -116,6 +117,7 @@ optional arguments:
preprocess, features, mwm, descriptions,
countries_txt, cleanup, index, ugc, popularity,
routing, routing_transit.
--only_coasts Build WorldCoasts.raw and WorldCoasts.rawgeom files
--production Build production maps. In another case, 'osm only
maps' are built - maps without additional data and
advertising.

14
tools/python/maps_generator/__main__.py
View file

@ -5,8 +5,8 @@ from argparse import ArgumentParser
from .generator import settings
from .generator.env import Env, find_last_build_dir, WORLDS_NAMES
from .generator.exceptions import ContinueError, SkipError, ValidationError
from .maps_generator import (start, reset_to_stage, ALL_STAGES,
stage_download_production_external,
from .maps_generator import (generate_maps, generate_coasts, reset_to_stage,
ALL_STAGES, stage_download_production_external,
stage_descriptions, stage_ugc, stage_popularity,
stages_as_string)
from .utils.collections import unique
@ -48,6 +48,11 @@ def parse_options():
default="",
help=f"Stage from which maps will be rebuild. Available stages: "
f"{', '.join([s.replace('stage_', '') for s in ALL_STAGES])}.")
parser.add_argument(
"--only_coasts",
default=False,
action="store_true",
help="Build WorldCoasts.raw and WorldCoasts.rawgeom files")
parser.add_argument(
"--production",
default=False,
@ -136,7 +141,10 @@ def main():
env = Env(options)
if env.from_stage:
reset_to_stage(env.from_stage, env)
start(env)
if env.only_coasts:
generate_coasts(env)
else:
generate_maps(env)
env.finish()

8
tools/python/maps_generator/generator/coastline.py
View file

@ -38,11 +38,3 @@ def make_coastline(env):
user_resource_path=env.user_resource_path,
make_coasts=True,
fail_on_coasts=True)
prefix = "WorldCoasts"
coastline_files = []
for f in os.listdir(env.coastline_path):
path = os.path.join(env.coastline_path, f)
if os.path.isfile(path) and f.startswith(prefix):
coastline_files.append(path)
return coastline_files

4
tools/python/maps_generator/generator/settings.py
View file

@ -42,6 +42,7 @@ VERSION_FILE_NAME = "version.txt"
# External resources
PLANET_URL = "https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf"
PLANET_MD5_URL = PLANET_URL + ".md5"
PLANET_COASTS_URL = ""
UGC_URL = ""
HOTELS_URL = ""
POPULARITY_URL= ""
@ -110,6 +111,7 @@ os.makedirs(os.path.dirname(os.path.abspath(LOG_FILE_PATH)), exist_ok=True)
PLANET_URL = _get_opt_path(config, "External", "PLANET_URL", PLANET_URL)
PLANET_MD5_URL = _get_opt_path(config, "External", "PLANET_MD5_URL", PLANET_MD5_URL)
PLANET_COASTS_URL = _get_opt_path(config, "External", "PLANET_COASTS_URL", PLANET_COASTS_URL)
UGC_URL = _get_opt_path(config, "External", "UGC_URL", UGC_URL)
HOTELS_URL = _get_opt_path(config, "External", "HOTELS_URL", HOTELS_URL)
POPULARITY_URL = _get_opt_path(config, "External", "POPULARITY_URL", POPULARITY_URL)
@ -120,6 +122,8 @@ FOOD_TRANSLATIONS_URL = _get_opt(config, "External", "FOOD_TRANSLATIONS_URL",
PLANET_O5M = os.path.join(MAIN_OUT_PATH, PLANET + ".o5m")
PLANET_PBF = os.path.join(MAIN_OUT_PATH, PLANET + ".osm.pbf")
PLANET_COASTS_GEOM_URL = os.path.join(PLANET_COASTS_URL, "latest.geom")
PLANET_COASTS_RAWGEOM_URL = os.path.join(PLANET_COASTS_URL, "latest.rawgeom")
if DEBUG:
PLANET_URL = "http://osmz.ru/mwm/islands/islands.o5m"

32
tools/python/maps_generator/maps_generator.py
View file

@ -17,7 +17,7 @@ from .generator import settings
from .generator.decorators import stage, country_stage, country_stage_log
from .generator.env import (planet_lock_file, build_lock_file,
WORLD_COASTS_NAME, WORLD_NAME, WORLDS_NAMES)
from .generator.exceptions import (ContinueError,
from .generator.exceptions import (ContinueError, BadExitStatusError,
wait_and_raise_if_fail)
from .generator.gen_tool import run_gen_tool
from .utils.file import is_verified, download_file
@ -103,9 +103,22 @@ def stage_features(env):
@stage
def stage_coastline(env):
coastline_files = coastline.make_coastline(env)
for file in coastline_files:
shutil.copy2(file, env.intermediate_path)
coasts_geom = "WorldCoasts.geom"
coasts_rawgeom = "WorldCoasts.rawgeom"
try:
coastline.make_coastline(env)
except BadExitStatusError:
logger.info("Build costs failed. Try to download the costs...")
download_external({
settings.PLANET_COASTS_GEOM_URL:
os.path.join(env.coastline_path, coasts_geom),
settings.PLANET_COASTS_RAWGEOM_URL:
os.path.join(env.coastline_path, coasts_rawgeom)
})
for f in [coasts_geom, coasts_rawgeom]:
path = os.path.join(env.coastline_path, f)
shutil.copy2(path, env.intermediate_path)
@country_stage
@ -281,7 +294,7 @@ def reset_to_stage(stage_name, env):
f.write(main_status)
def start(env):
def generate_maps(env):
stage_download_external(env)
stage_download_production_external(env)
with FileLock(planet_lock_file(), timeout=1) as planet_lock:
@ -296,3 +309,12 @@ def start(env):
stage_descriptions(env)
stage_countries_txt(env)
stage_cleanup(env)
def generate_coasts(env):
with FileLock(planet_lock_file(), timeout=1) as planet_lock:
stage_update_planet(env)
with FileLock(build_lock_file(env.out_path), timeout=1):
stage_coastline(env)
planet_lock.release()
stage_cleanup(env)

1
tools/python/maps_generator/var/etc/map_generator.ini.default
View file

@ -26,6 +26,7 @@ OSM_TOOLS_PATH: ~/osmctools
[External]
# PLANET_URL:
# PLANET_MD5_URL:
# PLANET_COASTS_URL:
# UGC_URL:
# HOTELS_URL:
# POPULARITY_URL: