From 5fa9b895a74b1eac5d23cbffa1d224b14fc94aed Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Tue, 28 May 2019 15:11:32 +0300 Subject: [PATCH] Review fixes --- tools/python/maps_generator/README.md | 6 +- tools/python/maps_generator/__main__.py | 37 ++++- tools/python/maps_generator/generator/env.py | 9 +- .../maps_generator/generator/exceptions.py | 4 + .../maps_generator/generator/statistics.py | 35 +++-- .../var/etc/stats_types_config.txt | 133 ++++++++++-------- 6 files changed, 143 insertions(+), 81 deletions(-) diff --git a/tools/python/maps_generator/README.md b/tools/python/maps_generator/README.md index c09b5810ee..e934506097 100644 --- a/tools/python/maps_generator/README.md +++ b/tools/python/maps_generator/README.md @@ -141,7 +141,7 @@ If you are not from the maps.me team, then you do not need the option --producti To generate maps for the whole world you need 400 GB of hard disk space and a computer with more than 64 GB. ### Examples #### Non-standard planet -If I want to generate maps for Japan I must complete the following steps: +If you want to generate maps for Japan you must complete the following steps: 1. Open https://download.geofabrik.de/asia/japan.html and copy url of osm.pbf and md5sum files. 2. Edit ini file: ```sh @@ -161,10 +161,10 @@ PLANET_MD5_URL: https://download.geofabrik.de/asia/japan-latest.osm.pbf.md5 3. Run ```sh -python$ python3.6 -m maps_generator --countries="World, WorldCoasts, Japan_*" --skip_stage="update_planet" +python$ python3.6 -m maps_generator --countries="World, WorldCoasts, Japan_*" --skip="update_planet" ``` -We must skip the step of updating the planet, because it is a non-standard planet. +You must skip the step of updating the planet, because it is a non-standard planet. #### Rebuild stages: For example, you changed routing code in omim project and want to regenerate maps. You must have previous generation. You may regenerate from stage routing only for two mwms: diff --git a/tools/python/maps_generator/__main__.py b/tools/python/maps_generator/__main__.py index 97a55b9371..39f59692d5 100644 --- a/tools/python/maps_generator/__main__.py +++ b/tools/python/maps_generator/__main__.py @@ -1,6 +1,6 @@ import logging import os -from argparse import ArgumentParser +from argparse import ArgumentParser, RawDescriptionHelpFormatter from .generator import settings from .generator.env import Env, find_last_build_dir, WORLDS_NAMES @@ -15,9 +15,41 @@ from .utils.collections import unique logger = logging.getLogger("maps_generator") +examples = """Examples: +1) Non-standard planet + If you want to generate maps for Japan you must complete the following steps: + 1. Open https://download.geofabrik.de/asia/japan.html and copy url of osm.pbf + and md5sum files. + 2. Edit the ini file: + maps_generator$ vim var/etc/map_generator.ini + + ... + [Main] + ... + DEBUG: 0 + ... + [External] + PLANET_URL: https://download.geofabrik.de/asia/japan-latest.osm.pbf + PLANET_MD5_URL: https://download.geofabrik.de/asia/japan-latest.osm.pbf.md5 + ... + + 3. Run + python$ python3.6 -m maps_generator --countries="World, WorldCoasts, Japan_*" --skip="update_planet" + + You must skip the step of updating the planet, because it is a non-standard planet. +2) Rebuild stages: + For example, you changed routing code in omim project and want to regenerate maps. + You must have previous generation. You may regenerate from stage routing only for two mwms: + + python$ python3.6 -m maps_generator -c --from_stage="routing" --countries="Japan_Kinki Region_Osaka_Osaka, Japan_Chugoku Region_Tottori" +""" + + def parse_options(): parser = ArgumentParser(description="Tool for generation maps for maps.me " "application.", + epilog=examples, + formatter_class=RawDescriptionHelpFormatter, parents=[settings.parser]) parser.add_argument( "-c", @@ -134,7 +166,8 @@ def main(): if not options["production"]: options["skip"] += stages_as_string( stage_download_production_external, - stage_ugc, stage_popularity, + stage_ugc, + stage_popularity, stage_descriptions, stage_localads, stage_statistics diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 111d400ec5..daea4fdc0e 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -99,12 +99,11 @@ class Env: os.path.join(self.draft_path, "borders")) self.osm2ft_path = os.path.join(self.out_path, "osm2ft") - if os.path.isdir(self.osm2ft_path): - for x in os.listdir(self.osm2ft_path): - p = os.path.join(self.osm2ft_path, x) - if os.path.isfile(p) and x.endswith(".mwm.osm2ft"): - shutil.move(p, os.path.join(self.mwm_path, x)) self._create_if_not_exist(self.osm2ft_path) + for x in os.listdir(self.osm2ft_path): + p = os.path.join(self.osm2ft_path, x) + if os.path.isfile(p) and x.endswith(".mwm.osm2ft"): + shutil.move(p, os.path.join(self.mwm_path, x)) self.node_storage = settings.NODE_STORAGE self.user_resource_path = settings.USER_RESOURCE_PATH diff --git a/tools/python/maps_generator/generator/exceptions.py b/tools/python/maps_generator/generator/exceptions.py index 7aa40c0538..51b5897c96 100644 --- a/tools/python/maps_generator/generator/exceptions.py +++ b/tools/python/maps_generator/generator/exceptions.py @@ -25,6 +25,10 @@ class BadExitStatusError(MapsGeneratorError): pass +class ParseError(MapsGeneratorError): + pass + + def wait_and_raise_if_fail(p): if p.wait() != os.EX_OK: raise BadExitStatusError(f"The launch of {' '.join(p.args)} failed.") diff --git a/tools/python/maps_generator/generator/statistics.py b/tools/python/maps_generator/generator/statistics.py index 6051e97c51..bca18696c6 100644 --- a/tools/python/maps_generator/generator/statistics.py +++ b/tools/python/maps_generator/generator/statistics.py @@ -3,6 +3,8 @@ import os import datetime from collections import defaultdict +from .exceptions import ParseError + RE_STAT = re.compile(r"(?:\d+\. )?([\w:|-]+?)\|: " r"size = \d+; " @@ -37,7 +39,10 @@ def read_stat(f): def read_config(f): config = [] for line in f: - columns = [c.strip() for c in line.split(";", 2)] + l = line.strip() + if l.startswith("#") or not l: + continue + columns = [c.strip() for c in l.split(";", 2)] columns[0] = re.compile(columns[0]) columns[1] = columns[1].lower() config.append(columns) @@ -48,27 +53,31 @@ def process_stat(config, stats): result = {} for param in config: res = 0 - for typ in stats: - if param[0].match(typ["name"]): + for t in stats: + if param[0].match(t["name"]): if param[1] == "len": - res += typ["len"] + res += t["len"] elif param[1] == "area": - res += typ["area"] + res += t["area"] elif param[1] == "cnt_names": - res += typ["names"] + res += t["names"] else: - res += typ["cnt"] + res += t["cnt"] result[str(param[0]) + param[1]] = res return result -def format_res(res, typ): - if typ == "len": - unit = "м" - elif typ == "area": - unit = "м²" +def format_res(res, t): + unit = None + if t == "len": + unit = "m" + elif t == "area": + unit = "m²" + elif t == "cnt" or t == "cnt_names": + unit = "pc" else: - unit = "шт." + raise ParseError(f"Unknown type {t}.") + return res, unit diff --git a/tools/python/maps_generator/var/etc/stats_types_config.txt b/tools/python/maps_generator/var/etc/stats_types_config.txt index 3026e9f86e..30b210f402 100644 --- a/tools/python/maps_generator/var/etc/stats_types_config.txt +++ b/tools/python/maps_generator/var/etc/stats_types_config.txt @@ -1,59 +1,76 @@ -barrier-(fence|gate);len;Заборы -building;cnt;Здания +# This file is used to calculate statistics by type. +# File format: +# Regular expression of type;metric type;statistic name +# +# Regular expression of type +# Types you can find in omim/data/mapcss-mapping.csv. +# You must replace the character '|' with the '_' character. +# +# Metric type +# There can be three types of metrics: +# 1) cnt - to count the number of objects +# 2) line - to calculate the total length of objects +# 3) area - to calculate the total area of ​​objects +# +# Statistic name +# The name of the statistics will be written to the resulting file. + +barrier-(fence|gate);len;Fences +building;cnt;Building (amenity|shop|historic)-.*;cnt;POI -(amenity|shop|historic)-.*;cnt_names;POI c именами -amenity-(cafe|restaurant|fast_food).*;cnt;Кафе и рестораны -amenity-(pub|bar);cnt;Бары и пабы -amenity-kindergarten;cnt;Детские сады -amenity-(school|university|college);cnt;Школы и университеты -amenity-parking.*;cnt;Автостоянки -amenity-parking.*;area;Автостоянки -amenity-pharmacy;cnt;Аптеки -amenity-place_of_worship.*;cnt;Храмы -amenity-(hospital|doctors);cnt;Больницы и поликлиники -amenity-toilets;cnt;Туалеты -amenity-(waste_disposal|recycling);cnt;Мусорные баки -highway-(motorway|trunk|primary|secondary|tertiary|residential|unclassified|service|track|living_street)(_link)?(-.*)?;len;Автодорожная сеть -highway-(footway|path|pedestrian|steps).*;len;Пешеходные дорожки -highway-.*-bridge;len;Мосты -highway-.*-tunnel;len;Туннели -highway-(footway|path|steps)-bridge;len;Пешеходные мосты -highway-(footway|path|steps)-tunnel;len;Пешеходные туннели -highway-steps.*;len;Лестницы -highway-speed_camera;cnt;Камеры контроля скорости -internet_access-wlan;cnt;Точки доступа Wi-Fi -leisure-(pitch|stadium|playing_fields|track|sports_centre).*;cnt;Спортплощадки и комплексы -leisure-playground;cnt;Детские площадки -man_made-lighthouse;cnt;Маяки -man_made-windmill;cnt;Ветряные мельницы -man_made-pipeline.*;len;Трубопроводы -natural-beach;cnt;Пляжи -natural-tree;cnt;Отдельностоящие деревья -natural-waterfall;cnt;Водопады -piste:type.*;len;Лыжни -place-(city.*|town|village|hamlet);cnt;Населённые пункты -place-island;cnt;Острова -power-(minor_)?line.*;len;Линии электропередачи -power-(pole|tower);cnt;Опоры ЛЭП -railway-(rail|monorail|light_rail|narrow_gauge|preserved|siding|spur|yard|disused|incline).*;len;Железные дороги -railway-.*-(bridge|tunnel);len;Железнодорожные мосты и туннели -railway-(razed|abandoned).*;len;Снятые ветки ж/д -railway-narrow_gauge.*;len;Узкоколейные ж/д -railway-tram(-.*)?;len;Трамвайные пути -railway-(halt|station);cnt;Станции железной дороги -railway-subway.*;len;Линии метро -highway-bus_stop|railway-tram_stop;cnt;Остановки наземного транспорта -shop-bakery;cnt;Пекарни -shop-books;cnt;Книжные магазины -shop-clothes;cnt;Магазины одежды -shop-shoes;cnt;Магазины обуви -shop-(convenience|supermarket);cnt;Продуктовые магазины -shop-florist;cnt;Цветочные салоны -shop-(hairdresser|beauty);cnt;Парикмахерские и салоны красоты -tourism-(guest_house|hos?tel|motel);cnt;Гостиницы и хостелы -tourism-(attraction|viewpoint);cnt;Достопримечательности и точки обзора -waterway-(canal|river|stream)(-.*)?;len;Реки, каналы и ручьи -landuse-cemetery.*;area;Кладбища -leisure-park.*;area;Парки -natural-beach;area;Пляжи -sponsored-booking;cnt;Booking отели +(amenity|shop|historic)-.*;cnt_names;POI with names +amenity-(cafe|restaurant|fast_food).*;cnt;Cafes and restaurants +amenity-(pub|bar);cnt;Bars and pubs +amenity-kindergarten;cnt;Kindergartens +amenity-(school|university|college);cnt;Schools and universities +amenity-parking.*;cnt;Parking lots +amenity-parking.*;area;Parking lots +amenity-pharmacy;cnt;Pharmacies +amenity-place_of_worship.*;cnt;Temples +amenity-(hospital|doctors);cnt;Hospitals and clinics +amenity-toilets;cnt;Toilets +amenity-(waste_disposal|recycling);cnt;Garbage bins +highway-(motorway|trunk|primary|secondary|tertiary|residential|unclassified|service|track|living_street)(_link)?(-.*)?;len;Road network +highway-(footway|path|pedestrian|steps).*;len;Footpaths +highway-.*-bridge;len;Bridges +highway-.*-tunnel;len;Tunnels +highway-(footway|path|steps)-bridge;len;Pedestrian bridges +highway-(footway|path|steps)-tunnel;len;Pedestrian tunnels +highway-steps.*;len;Stairs +highway-speed_camera;cnt;Speed ​​cameras +internet_access-wlan;cnt;Wi-Fi access points +leisure-(pitch|stadium|playing_fields|track|sports_centre).*;cnt;Sports grounds and complexes +leisure-playground;cnt;Playgrounds +man_made-lighthouse;cnt;Lighthouses +man_made-windmill;cnt;Windmills +man_made-pipeline.*;len;Pipelines +natural-beach;cnt;Beaches +natural-tree;cnt;Trees +natural-waterfall;cnt;Waterfalls +piste:type.*;len;Ski trails +place-(city.*|town|village|hamlet);cnt;Settlements +place-island;cnt;Islands +power-(minor_)?line.*;len;Power lines +power-(pole|tower);cnt;Power Line Supports +railway-(rail|monorail|light_rail|narrow_gauge|preserved|siding|spur|yard|disused|incline).*;len;Railways +railway-.*-(bridge|tunnel);len;Railway bridges and tunnels +railway-(razed|abandoned).*;len;Abandoned railways +railway-narrow_gauge.*;len;Narrow gauge railways +railway-tram(-.*)?;len;Tram rails +railway-(halt|station);cnt;Railway stations +railway-subway.*;len;Subway lines +highway-bus_stop|railway-tram_stop;cnt;Ground Transportation Stops +shop-bakery;cnt;Bakeries +shop-books;cnt;Book stores +shop-clothes;cnt;Clothing stores +shop-shoes;cnt;Shoe stores +shop-(convenience|supermarket);cnt;Grocery stores +shop-florist;cnt;Flower shops +shop-(hairdresser|beauty);cnt;Hairdressers and beauty salons +tourism-(guest_house|hos?tel|motel);cnt;Hotels and hostels +tourism-(attraction|viewpoint);cnt;Attractions and viewpoints +waterway-(canal|river|stream)(-.*)?;len;Rivers, canals and streams +landuse-cemetery.*;area;Cemeteries +leisure-park.*;area;Parks +natural-beach;area;Beaches +sponsored-booking;cnt;Booking hotels