From cc0fee616702be572bd6eebdceb2d6851349a14a Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov <35913079+alexey-zakharenkov@users.noreply.github.com> Date: Tue, 20 Apr 2021 00:24:41 +0300 Subject: [PATCH 01/17] Update subway_structure.py --- subway_structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subway_structure.py b/subway_structure.py index 4a6de18..c5fc18e 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -7,7 +7,7 @@ from css_colours import normalize_colour from collections import Counter, defaultdict -SPREADSHEET_ID = '1-UHDzfBwHdeyFxgC5cE_MaNQotF3-Y0r1nW9IwpIEj8' +SPREADSHEET_ID = '1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k' MAX_DISTANCE_TO_ENTRANCES = 300 # in meters MAX_DISTANCE_STOP_TO_LINE = 50 # in meters ALLOWED_STATIONS_MISMATCH = 0.02 # part of total station count -- 2.45.3 From 724a9344a97dc0784b4e392b643d4166aebf48fa Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Mon, 7 Jun 2021 18:10:45 +0300 Subject: [PATCH 02/17] Add 'POLY' parameter to bash script and a script to generate a *.poly file with metro bbox union --- make_all_metro_poly.py | 33 +++++++++++++++++++++++ scripts/process_subways.sh | 55 ++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 20 deletions(-) create mode 100644 make_all_metro_poly.py diff --git a/make_all_metro_poly.py b/make_all_metro_poly.py new file mode 100644 index 0000000..4276b2b --- /dev/null +++ b/make_all_metro_poly.py @@ -0,0 +1,33 @@ +import shapely.geometry +import shapely.ops + +from process_subways import download_cities + + +def make_disjoint_metro_polygons(): + cities = download_cities() + + polygons = [] + for c in cities: + polygon = shapely.geometry.Polygon([ + (c.bbox[1], c.bbox[0]), + (c.bbox[1], c.bbox[2]), + (c.bbox[3], c.bbox[2]), + (c.bbox[3], c.bbox[0]), + ]) + polygons.append(polygon) + + union = shapely.ops.unary_union(polygons) + + print("all metro") + for i, polygon in enumerate(union, start=1): + assert len(polygon.interiors) == 0 + print(i) + for point in polygon.exterior.coords: + print(" {lon} {lat}".format(lon=point[0], lat=point[1])) + print("END") + print("END") + + +if __name__ == '__main__': + make_disjoint_metro_polygons() diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 2d0a5b5..1031c21 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -3,16 +3,25 @@ set -e -u if [ $# -lt 1 -a -z "${PLANET-}" ]; then echo "This script updates a planet or an extract, processes metro networks in it" - echo "and produses a set of HTML files with validation results." + echo "and produces a set of HTML files with validation results." echo echo "Usage: $0 " echo - echo "Variable reference:" + echo "Enveronment variable reference:" echo "- PLANET: path for the source o5m file (the entire planet or an extract)" echo "- CITY: name of a city to process" echo "- BBOX: bounding box of an extract; x1,y1,x2,y2" - echo "- DUMP: file name to dump city data" + echo "- POLY: *.poly file name with bounding [multi]polygon of an extract" + echo "- SKIP_PLANET_UPDATE: skip \$PLANET file update. Any non-empty string is True" + echo "- SKIP_FILTERING: skip filtering railway data. Any non-empty string is True" + echo "- FILTERED_DATA: path to filtered data. Defaults to \$TMPDIR/subways.osm" echo "- MAPSME: file name for maps.me json output" + echo "- DUMP: directory/file name to dump YAML city data. Do not set to omit dump" + echo "- GEOJSON: directory/file name to dump GeoJSON data. Do not set to omit dump" + echo "- ELEMENTS_CACHE: file name to elements cache. Allows OSM xml processing phase" + echo "- CITY_CACHE: json file with good cities obtained on previous validation runs" + echo "- RECOVERY_PATH: file with some data collected at previous validation runs that" + echo " may help to recover some simple validation errors" echo "- OSMCTOOLS: path to osmconvert and osmupdate binaries" echo "- PYTHON: python 3 executable" echo "- GIT_PULL: set to 1 to update the scripts" @@ -55,31 +64,37 @@ if [ -n "${GIT_PULL-}" ]; then ( # Updating the planet file -PLANET_ABS="$(cd "$(dirname "$PLANET")"; pwd)/$(basename "$PLANET")" -pushd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory -OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m "$PLANET_ABS" ${BBOX+"-b=$BBOX"} "$PLANET_ABS.new.o5m" 2>&1) -if [ -n "$OSMUPDATE_ERRORS" ]; then - echo "osmupdate failed: $OSMUPDATE_ERRORS" - exit 5 +if [ "${SKIP_PLANET_UPDATE:-not_defined}" == "not_defined" ]; then + PLANET_ABS="$(cd "$(dirname "$PLANET")"; pwd)/$(basename "$PLANET")" + pushd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory + OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} "$PLANET_ABS" "$PLANET_ABS.new.o5m" 2>&1) + if [ -n "$OSMUPDATE_ERRORS" ]; then + echo "osmupdate failed: $OSMUPDATE_ERRORS" + exit 5 + fi + popd + mv "$PLANET_ABS.new.o5m" "$PLANET_ABS" fi -popd -mv "$PLANET_ABS.new.o5m" "$PLANET_ABS" - # Filtering it -FILTERED_DATA="$TMPDIR/subways.osm" -QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group" -QNODES="railway=station station=subway =light_rail =monorail railway=subway_entrance subway=yes light_rail=yes monorail=yes train=yes" -"$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-relations="$QRELATIONS" --keep-nodes="$QNODES" --drop-author -o="$FILTERED_DATA" +if [ "${FILTERED_DATA:-not_defined}" == "not_defined" ]; then + FILTERED_DATA="$TMPDIR/subways.osm" +fi + +if [ "${SKIP_FILTERING:-not_defined}" == "not_defined" ]; then + QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group" + QNODES="railway=station station=subway =light_rail =monorail railway=subway_entrance subway=yes light_rail=yes monorail=yes train=yes" + "$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-relations="$QRELATIONS" --keep-nodes="$QNODES" --drop-author -o="$FILTERED_DATA" +fi # Running the validation VALIDATION="$TMPDIR/validation.json" -"$PYTHON" "$SUBWAYS_PATH/process_subways.py" -q -x "$FILTERED_DATA" -l "$VALIDATION" ${MAPSME+-o "$MAPSME"}\ - ${CITY+-c "$CITY"} ${DUMP+-d "$DUMP"} ${GEOJSON+-j "$GEOJSON"}\ - ${ELEMENTS_CACHE+-i "$ELEMENTS_CACHE"} ${CITY_CACHE+--cache "$CITY_CACHE"}\ - ${RECOVERY_PATH+-r "$RECOVERY_PATH"} +"$PYTHON" "$SUBWAYS_PATH/process_subways.py" -q -x "$FILTERED_DATA" -l "$VALIDATION" ${MAPSME:+-o "$MAPSME"}\ + ${CITY:+-c "$CITY"} ${DUMP:+-d "$DUMP"} ${GEOJSON:+-j "$GEOJSON"}\ + ${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} ${CITY_CACHE:+--cache "$CITY_CACHE"}\ + ${RECOVERY_PATH:+-r "$RECOVERY_PATH"} rm "$FILTERED_DATA" # Preparing HTML files -- 2.45.3 From 4884c711c29c543ec2cc14f2fac0939700461342 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Mon, 7 Jun 2021 18:11:13 +0300 Subject: [PATCH 03/17] mapsme_json_to_cities.py script --- mapsme_json_to_cities.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 mapsme_json_to_cities.py diff --git a/mapsme_json_to_cities.py b/mapsme_json_to_cities.py new file mode 100644 index 0000000..cac7946 --- /dev/null +++ b/mapsme_json_to_cities.py @@ -0,0 +1,36 @@ +""" +This script generates a list of good/all network names. +It is used by subway render to generate the list of network at frontend. +It uses two sources: a mapsme.json validator output with good networks, and +a google spreadsheet with networks for the process_subways.download_cities() +function. +""" + +import json +import sys + +from process_subways import download_cities + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('Usage: {} [--with-bad]'.format(sys.argv[0])) + sys.exit(1) + + with_bad = len(sys.argv) > 2 and sys.argv[2] == '--with-bad' + + subway_json_file = sys.argv[1] + with open(subway_json_file) as f: + subway_json = json.load(f) + good_cities = set(n.get('network', n.get('title')) for n in subway_json['networks']) + + cities = download_cities() + lines = [] + for c in cities: + if c.name in good_cities: + lines.append(f"{c.name}, {c.country}") + elif with_bad: + lines.append(f"{c.name}, {c.country} (Bad)") + + for line in sorted(lines): + print(line) -- 2.45.3 From 44c6dc77e4257a1dc70793da14c6a26309e59cd7 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Thu, 17 Jun 2021 22:53:00 +0300 Subject: [PATCH 04/17] Elaborate planet processing pipeline --- scripts/process_subways.sh | 258 ++++++++++++++++++++++++++++--------- 1 file changed, 196 insertions(+), 62 deletions(-) diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 1031c21..1376955 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -1,101 +1,235 @@ #!/bin/bash set -e -u -if [ $# -lt 1 -a -z "${PLANET-}" ]; then - echo "This script updates a planet or an extract, processes metro networks in it" - echo "and produces a set of HTML files with validation results." - echo - echo "Usage: $0 " - echo - echo "Enveronment variable reference:" - echo "- PLANET: path for the source o5m file (the entire planet or an extract)" - echo "- CITY: name of a city to process" - echo "- BBOX: bounding box of an extract; x1,y1,x2,y2" - echo "- POLY: *.poly file name with bounding [multi]polygon of an extract" - echo "- SKIP_PLANET_UPDATE: skip \$PLANET file update. Any non-empty string is True" - echo "- SKIP_FILTERING: skip filtering railway data. Any non-empty string is True" - echo "- FILTERED_DATA: path to filtered data. Defaults to \$TMPDIR/subways.osm" - echo "- MAPSME: file name for maps.me json output" - echo "- DUMP: directory/file name to dump YAML city data. Do not set to omit dump" - echo "- GEOJSON: directory/file name to dump GeoJSON data. Do not set to omit dump" - echo "- ELEMENTS_CACHE: file name to elements cache. Allows OSM xml processing phase" - echo "- CITY_CACHE: json file with good cities obtained on previous validation runs" - echo "- RECOVERY_PATH: file with some data collected at previous validation runs that" - echo " may help to recover some simple validation errors" - echo "- OSMCTOOLS: path to osmconvert and osmupdate binaries" - echo "- PYTHON: python 3 executable" - echo "- GIT_PULL: set to 1 to update the scripts" - echo "- TMPDIR: path to temporary files" - echo "- HTML_DIR: target path for generated HTML files" - echo "- SERVER: server name and path to upload HTML files (e.g. ilya@osmz.ru:/var/www/)" - echo "- SERVER_KEY: rsa key to supply for uploading the files" - echo "- REMOVE_HTML: set to 1 to remove HTML_DIR after uploading" - exit 1 +if [ $# -gt 0 -a \( "${1-}" = "-h" -o "${1-}" = '--help' \) ]; then + cat << EOF +This script updates a planet or an extract, processes metro networks in it +and produces a set of HTML files with validation results. + +Usage: $0 + +In more detail, the script does the following: + - If \$PLANET is a remote file, downloads it. + - Unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro. + - Makes a *.o5m extract of the \$PLANET using the *.poly file. + - Updates the extract. + - Filters railway infrastructure from the extract. + - Uses filtered file for validation and generates other output files. + - Copies results onto remove server, if it is set up. + +During this procedure, as many steps are skipped as possible. Namely: + - Making metro extract is skipped if \$PLANET_METRO variable is set and the file exists. + - Update with osmupdate is skipped if SKIP_PLANET_UPDATE or \$SKIP_FILTERING is set. + - Filtering is skipped if \$SKIP_FILTERING is set and \$FILTERED_DATA is set and the file exists. + +Generated files \$POLY, \$PLANET_METRO, \$FILTERED_DATA are deleted if the correponding variable is not defined or is null, otherwise they are kept. +The PLANET file from remote URL is saved to a tempfile and is removed at the end. + +Environment variable reference: + - PLANET: path to a local or remote o5m or pbf source file (the entire planet or an extract) + - PLANET_METRO: path to local o5m file with extract of cities having metro. + It's used instead of \$PLANET if exists otherwise it's created first + - CITY: name of a city/country to process + - BBOX: bounding box of an extract; x1,y1,x2,y2. Has precedence over \$POLY + - POLY: *.poly file with [multi]polygon comprising cities with metro. + If neither \$BBOX nor \$POLY is set, then \$POLY is generated + - SKIP_PLANET_UPDATE: skip \$PLANET file update. Any non-empty string is True + - SKIP_FILTERING: skip filtering railway data. Any non-empty string is True + - FILTERED_DATA: path to filtered data. Defaults to \$TMPDIR/subways.osm + - MAPSME: file name for maps.me json output + - DUMP: directory/file name to dump YAML city data. Do not set to omit dump + - GEOJSON: directory/file name to dump GeoJSON data. Do not set to omit dump + - ELEMENTS_CACHE: file name to elements cache. Allows OSM xml processing phase + - CITY_CACHE: json file with good cities obtained on previous validation runs + - RECOVERY_PATH: file with some data collected at previous validation runs that + may help to recover some simple validation errors + - OSMCTOOLS: path to osmconvert and osmupdate binaries + - PYTHON: python 3 executable + - GIT_PULL: set to 1 to update the scripts + - TMPDIR: path to temporary files + - HTML_DIR: target path for generated HTML files + - SERVER: server name and path to upload HTML files (e.g. ilya@osmz.ru:/var/www/) + - SERVER_KEY: rsa key to supply for uploading the files + - REMOVE_HTML: set to 1 to remove \$HTML_DIR after uploading +EOF + exit fi -[ -n "${WHAT-}" ] && echo WHAT -PLANET="${PLANET:-${1-}}" -[ ! -f "$PLANET" ] && echo "Cannot find planet file $PLANET" && exit 2 -OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" -if [ ! -f "$OSMCTOOLS/osmupdate" ]; then - if which osmupdate > /dev/null; then - OSMCTOOLS="$(dirname "$(which osmupdate)")" - else - echo "Please compile osmctools to $OSMCTOOLS" - exit 3 +function check_osmctools() { + OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" + if [ ! -f "$OSMCTOOLS/osmupdate" ]; then + if which osmupdate > /dev/null; then + OSMCTOOLS="$(dirname "$(which osmupdate)")" + else + echo "Please compile osmctools to $OSMCTOOLS" + exit 1 + fi fi -fi +} + + +function check_poly() { + # Checks or generates *.poly file with city bboxes where + # there is a metro but only once during script run + + if [ -z "${POLY_CHECKED-}" ]; then + if [ -n "${BBOX-}" ]; then + # If BBOX is set, then exclude POLY at all from processing + POLY="" + else + if [ -z "${POLY-}" ]; then + NEED_TO_REMOVE_POLY=1 + fi + + if [ -z "${POLY-}" -o ! -f "${POLY-}" ]; then + POLY=${POLY:-$(mktemp "$TMPDIR/all-metro.XXXXXXXX.poly")} + if [ -n "$("$PYTHON" -c "import shapely" 2>&1)" ]; then + "$PYTHON" -m pip install shapely + fi + "$PYTHON" "$SUBWAYS_PATH"/make_all_metro_poly.py > "$POLY" + fi + fi + POLY_CHECKED=1 + fi +} + + PYTHON=${PYTHON:-python3} # This will fail if there is no python "$PYTHON" --version > /dev/null + SUBWAYS_PATH="$(dirname "$0")/.." -[ ! -f "$SUBWAYS_PATH/process_subways.py" ] && echo "Please clone the subways repo to $SUBWAYS_PATH" && exit 4 +if [ ! -f "$SUBWAYS_PATH/process_subways.py" ]; then + echo "Please clone the subways repo to $SUBWAYS_PATH" + exit 2 +fi + TMPDIR="${TMPDIR:-$SUBWAYS_PATH}" # Downloading the latest version of the subways script - - if [ -n "${GIT_PULL-}" ]; then ( cd "$SUBWAYS_PATH" git pull origin master ) fi +if [ -z "${FILTERED_DATA-}" ]; then + FILTERED_DATA="$TMPDIR/subways.osm" + NEED_TO_REMOVE_FILTERED_DATA=1 +fi -# Updating the planet file +if [ -z "${SKIP_FILTERING-}" -o ! -f "$FILTERED_DATA" ]; then + NEED_FILTER=1 +fi -if [ "${SKIP_PLANET_UPDATE:-not_defined}" == "not_defined" ]; then - PLANET_ABS="$(cd "$(dirname "$PLANET")"; pwd)/$(basename "$PLANET")" + +if [ -n "${NEED_FILTER-}" ]; then + + # If $PLANET_METRO file doesn't exist, create it + + if [ -n "${PLANET_METRO-}" ]; then + EXT=${PLANET_METRO##*.} + if [ ! "$EXT" = "osm" -a ! "$EXT" == "xml" -a ! "$EXT" = "o5m" ]; then + echo "Only o5m/xml/osm file formats are supported for filtering." + exit 3 + fi + fi + + if [ ! -f "${PLANET_METRO-}" ]; then + check_osmctools + check_poly + + PLANET="${PLANET:-${1-}}" + EXT="${PLANET##*.}" + if [ ! "$EXT" = "pbf" -a ! "$EXT" = "o5m" ]; then + echo "Cannot process '$PLANET' planet file." + echo "Only pbf/o5m source planet files are supported." + exit 4 + fi + + if [ "${PLANET:0:7}" = "http://" -o \ + "${PLANET:0:8}" = "https://" -o \ + "${PLANET:0:6}" = "ftp://" ]; then + PLANET_TEMP=$(mktemp "$TMPDIR/planet.XXXXXXXX.$EXT") + wget -O "$PLANET_TEMP" "$PLANET" + PLANET="$PLANET_TEMP" + elif [ ! -f "$PLANET" ]; then + echo "Cannot find planet file '$PLANET'"; + exit 5 + fi + + if [ -z "${PLANET_METRO-}" ]; then + PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.o5m") + NEED_TO_REMOVE_PLANET_METRO=1 + fi + + if [ "$PLANET" = "$PLANET_METRO" ]; then + echo "PLANET_METRO parameter shouldn't point to PLANET." + exit 6 + fi + + "$OSMCTOOLS"/osmconvert "$PLANET" \ + ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} -o="$PLANET_METRO" + fi +fi + +if [ -n "${PLANET_TEMP-}" ]; then + rm "$PLANET_TEMP" +fi + +# Updating the planet-metro file + +# If there's no need to filter, then update is also unnecessary +if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then + check_osmctools + check_poly + PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")" pushd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory - OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} "$PLANET_ABS" "$PLANET_ABS.new.o5m" 2>&1) + OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \ + ${POLY:+"-B=$POLY"} "$PLANET_METRO_ABS" \ + "$PLANET_METRO_ABS.new.o5m" 2>&1) if [ -n "$OSMUPDATE_ERRORS" ]; then echo "osmupdate failed: $OSMUPDATE_ERRORS" - exit 5 + exit 7 fi popd - mv "$PLANET_ABS.new.o5m" "$PLANET_ABS" + mv "$PLANET_METRO_ABS.new.o5m" "$PLANET_METRO_ABS" fi -# Filtering it +# Filtering planet-metro -if [ "${FILTERED_DATA:-not_defined}" == "not_defined" ]; then - FILTERED_DATA="$TMPDIR/subways.osm" -fi - -if [ "${SKIP_FILTERING:-not_defined}" == "not_defined" ]; then +if [ -n "${NEED_FILTER-}" ]; then QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group" QNODES="railway=station station=subway =light_rail =monorail railway=subway_entrance subway=yes light_rail=yes monorail=yes train=yes" - "$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-relations="$QRELATIONS" --keep-nodes="$QNODES" --drop-author -o="$FILTERED_DATA" + "$OSMCTOOLS/osmfilter" "$PLANET_METRO" \ + --keep= \ + --keep-relations="$QRELATIONS" \ + --keep-nodes="$QNODES" \ + --drop-author \ + -o="$FILTERED_DATA" +fi + +if [ -n "${NEED_TO_REMOVE_PLANET_METRO-}" ]; then + rm $PLANET_METRO +fi +if [ -n "${NEED_TO_REMOVE_POLY-}" ]; then + rm $POLY fi # Running the validation VALIDATION="$TMPDIR/validation.json" -"$PYTHON" "$SUBWAYS_PATH/process_subways.py" -q -x "$FILTERED_DATA" -l "$VALIDATION" ${MAPSME:+-o "$MAPSME"}\ - ${CITY:+-c "$CITY"} ${DUMP:+-d "$DUMP"} ${GEOJSON:+-j "$GEOJSON"}\ - ${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} ${CITY_CACHE:+--cache "$CITY_CACHE"}\ +"$PYTHON" "$SUBWAYS_PATH/process_subways.py" -q \ + -x "$FILTERED_DATA" -l "$VALIDATION" \ + ${MAPSME:+-o "$MAPSME"} \ + ${CITY:+-c "$CITY"} ${DUMP:+-d "$DUMP"} ${GEOJSON:+-j "$GEOJSON"} \ + ${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} \ + ${CITY_CACHE:+--cache "$CITY_CACHE"} \ ${RECOVERY_PATH:+-r "$RECOVERY_PATH"} -rm "$FILTERED_DATA" + +if [ -n "${NEED_TO_REMOVE_FILTERED_DATA-}" ]; then + rm "$FILTERED_DATA" +fi # Preparing HTML files @@ -107,7 +241,6 @@ fi mkdir -p $HTML_DIR rm -f "$HTML_DIR"/*.html "$PYTHON" "$SUBWAYS_PATH/validation_to_html.py" "$VALIDATION" "$HTML_DIR" -rm "$VALIDATION" # Uploading files to the server @@ -117,3 +250,4 @@ if [ -n "${SERVER-}" ]; then rm -r "$HTML_DIR" fi fi + -- 2.45.3 From 2734f9fd9e5d4e7dd2eeb8fbbf7c77d157225140 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Fri, 18 Jun 2021 01:10:59 +0300 Subject: [PATCH 05/17] Update README --- README.md | 83 +++++++++++++++++++++++++++++--------- scripts/process_subways.sh | 24 ++++++----- 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index c1edfa3..31a96fa 100644 --- a/README.md +++ b/README.md @@ -5,44 +5,86 @@ systems in the world from OpenStreetMap. `subway_structure.py` produces a list of disjunct systems that can be used for routing and for displaying of metro maps. -## How To Validate +## How To Validate * Choose transport data source: 1. Download or update a planet file in o5m format (using `osmconvert` and `osmupdate`). Run `osmfilter` to extract a portion of data for all subways. Or 2. If you don't specify `--xml` or `--source` option to the `process_subways.py` script - it tries to fetch data over [Overpass API](https://wiki.openstreetmap.org/wiki/Overpass_API). **Not suitable for whole planet or large countries.** + it tries to fetch data over [Overpass API](https://wiki.openstreetmap.org/wiki/Overpass_API). + **Not suitable for the whole planet or large countries.** * Run `process_subways.py` with appropriate set of command line arguments to build metro structures and receive a validation log. * Run `validation_to_html.py` on that log to create readable HTML tables. -## Validation Script -There is a `process_subways.sh` in the `scripts` directory. The author uses it for -updating both the planet and a city he's working on. Here is an example of a script -for updating the London Underground network: +## Validating of all metro networks + +There is a `process_subways.sh` in the `scripts` directory that is suitable +for validation of all or many metro networks. It relies on a bunch of +environment variables and takes advantage of previous validation runs +for effective recurring validations. See +```bash +./scripts/process_subways.sh --help +``` +for details. Here is an example of the script usage: ```bash -PLANET_PATH=$HOME/osm/planet -export OSMCTOOLS="$PLANET_PATH" -export PLANET="$PLANET_PATH/london.o5m" -export HTML_DIR=tmp_html -export BBOX=-0.681152,51.286758,0.334015,51.740636 -export CITY="London" -export DUMP=london.yaml +export PLANET=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf +export PLANET_METRO="$HOME/metro/planet-metro.o5m +export OSMCTOOLS="$HOME/osmctools" +export TMPDIR="$HOME/metro/tmp" +export HTML_DIR="$HOME/metro/tmp_html" +export DUMP="$HTML_DIR" scripts/process_subways.sh ``` -The bounding box can be found in the -[Google Spreadsheet](https://docs.google.com/spreadsheets/d/1-UHDzfBwHdeyFxgC5cE_MaNQotF3-Y0r1nW9IwpIEj8/edit?usp=sharing). +Set the PLANET_METRO variable to avoid the whole planet processing each time. +Delete the file (but not the variable) to re-generate it if a new city has been added or +a city's bbox has been extended. -This can be simplified by using the `build_city.sh` script, which fetches the bbox from the web: - scripts/build_city.sh london.o5m London +## Validating of a single city + +A single city or a country with few metro networks can be validated much faster +if you allow the `process_subway.py` to fetch data from Overpass API. Here are the steps: + +1. Python3 interpreter required (3.5+) +2. Clone the repo + ``` + git clone https://github.com/alexey-zakharenkov/subways.git subways_validator + cd subways_validator + ``` +3. Execute + ```bash + python3 ./process_subways.py --bbox -c "London" \ + -l validation.log -d London.yaml + ``` + here + - `-c` stands for "city" i.e. network name from the google spreadsheet + - `-l` - path to validation log file + - `-d` (optional) - path to dump network info in YAML format + - `-i` (optional) - path to save overpass-api JSON response + - `-j` (optional) - path to output network GeoJSON (used for rendering) + + `validation.log` would contain the list of errors and warnings. + To convert it into pretty HTML format +4. do + ```bash + mkdir html + python3 ./validation_to_html.py validation.log html + ``` + +## Related external resources + +Summary information about all metro networks that are monitored is gathered in the +[Google Spreadsheet](https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k). + +Not so regular updates of validation results are available at +[this website](https://alexey-zakharenkov.github.io/subways/rapid/). -Daily updates of validation results are available at [this website](http://osm-subway.maps.me). ## Adding Stop Areas To OSM @@ -53,4 +95,7 @@ just upload it. ## Author and License -All scripts were written by Ilya Zverev for MAPS.ME. Published under Apache Licence 2.0. +The main scripts were originally written by Ilya Zverev for MAPS.ME +and were published under Apache Licence 2.0 at https://github.com/mapsme/subways/. + +This fork is maintained by Alexey Zakharenkov and is also published under Apache Licence 2.0. diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 1376955..8a02442 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -6,32 +6,34 @@ if [ $# -gt 0 -a \( "${1-}" = "-h" -o "${1-}" = '--help' \) ]; then This script updates a planet or an extract, processes metro networks in it and produces a set of HTML files with validation results. -Usage: $0 +Usage: $0 [] In more detail, the script does the following: - If \$PLANET is a remote file, downloads it. - - Unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro. + - If \$BBOX variable is set, proceeds with this setting for the planet clipping. Otherwise uses \$POLY: + unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro. - Makes a *.o5m extract of the \$PLANET using the *.poly file. - Updates the extract. - Filters railway infrastructure from the extract. - - Uses filtered file for validation and generates other output files. - - Copies results onto remove server, if it is set up. + - Uses filtered file for validation and generates a bunch of output files. + - Copies results onto remote server, if it is set up. During this procedure, as many steps are skipped as possible. Namely: - Making metro extract is skipped if \$PLANET_METRO variable is set and the file exists. - - Update with osmupdate is skipped if SKIP_PLANET_UPDATE or \$SKIP_FILTERING is set. + - Update with osmupdate is skipped if \$SKIP_PLANET_UPDATE or \$SKIP_FILTERING is set. - Filtering is skipped if \$SKIP_FILTERING is set and \$FILTERED_DATA is set and the file exists. -Generated files \$POLY, \$PLANET_METRO, \$FILTERED_DATA are deleted if the correponding variable is not defined or is null, otherwise they are kept. -The PLANET file from remote URL is saved to a tempfile and is removed at the end. +Generated files \$POLY, \$PLANET_METRO, \$FILTERED_DATA are deleted if the corresponding +variable is not defined or is null, otherwise they are kept. +The \$PLANET file from remote URL is saved to a tempfile and is removed at the end. Environment variable reference: - PLANET: path to a local or remote o5m or pbf source file (the entire planet or an extract) - - PLANET_METRO: path to local o5m file with extract of cities having metro. + - PLANET_METRO: path to a local o5m file with extract of cities having metro It's used instead of \$PLANET if exists otherwise it's created first - CITY: name of a city/country to process - BBOX: bounding box of an extract; x1,y1,x2,y2. Has precedence over \$POLY - - POLY: *.poly file with [multi]polygon comprising cities with metro. + - POLY: *.poly file with [multi]polygon comprising cities with metro If neither \$BBOX nor \$POLY is set, then \$POLY is generated - SKIP_PLANET_UPDATE: skip \$PLANET file update. Any non-empty string is True - SKIP_FILTERING: skip filtering railway data. Any non-empty string is True @@ -70,8 +72,8 @@ function check_osmctools() { function check_poly() { - # Checks or generates *.poly file with city bboxes where - # there is a metro but only once during script run + # Checks or generates *.poly file covering cities where + # there is a metro; does this only once during script run. if [ -z "${POLY_CHECKED-}" ]; then if [ -n "${BBOX-}" ]; then -- 2.45.3 From 2dfb2e46e0b1105ae5730499e4893014fcc56f39 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Fri, 25 Jun 2021 23:54:55 +0300 Subject: [PATCH 06/17] Get rid of obsolete --bbox parameter for process_subways.py --- README.md | 2 +- process_subways.py | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 31a96fa..89ae9cb 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ if you allow the `process_subway.py` to fetch data from Overpass API. Here are t ``` 3. Execute ```bash - python3 ./process_subways.py --bbox -c "London" \ + python3 ./process_subways.py -c "London" \ -l validation.log -d London.yaml ``` here diff --git a/process_subways.py b/process_subways.py index 401a312..806b150 100755 --- a/process_subways.py +++ b/process_subways.py @@ -26,13 +26,11 @@ from subway_structure import ( ) -def overpass_request(overground, overpass_api, bboxes=None): +def overpass_request(overground, overpass_api, bboxes): query = '[out:json][timeout:1000];(' - if bboxes is None: - bboxes = [None] modes = MODES_OVERGROUND if overground else MODES_RAPID for bbox in bboxes: - bbox_part = '' if not bbox else '({})'.format(','.join(str(coord) for coord in bbox)) + bbox_part = '({})'.format(','.join(str(coord) for coord in bbox)) query += '(' for mode in modes: query += 'rel[route="{}"]{};'.format(mode, bbox_part) @@ -52,13 +50,12 @@ def overpass_request(overground, overpass_api, bboxes=None): def multi_overpass(overground, overpass_api, bboxes): - if not bboxes: - return overpass_request(overground, overpass_api, None) SLICE_SIZE = 10 + INTERREQUEST_WAIT = 5 # in seconds result = [] for i in range(0, len(bboxes) + SLICE_SIZE - 1, SLICE_SIZE): if i > 0: - time.sleep(5) + time.sleep(INTERREQUEST_WAIT) result.extend(overpass_request(overground, overpass_api, bboxes[i:i+SLICE_SIZE])) return result @@ -167,9 +164,6 @@ if __name__ == '__main__': parser.add_argument('--overpass-api', default='http://overpass-api.de/api/interpreter', help="Overpass API URL") - parser.add_argument( - '-b', '--bbox', action='store_true', - help='Use city boundaries to query Overpass API instead of querying the world') parser.add_argument('-q', '--quiet', action='store_true', help='Show only warnings and errors') parser.add_argument('-c', '--city', help='Validate only a single city or a country') parser.add_argument('-t', '--overground', action='store_true', @@ -232,10 +226,7 @@ if __name__ == '__main__': logging.error('Would not download that many cities from Overpass API, ' 'choose a smaller set') sys.exit(3) - if options.bbox: - bboxes = [c.bbox for c in cities] - else: - bboxes = None + bboxes = [c.bbox for c in cities] logging.info('Downloading data from Overpass API') osm = multi_overpass(options.overground, options.overpass_api, bboxes) calculate_centers(osm) -- 2.45.3 From b422ccb6adb8d2b374a4277f46e00bd520e49495 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Sat, 26 Jun 2021 00:41:11 +0300 Subject: [PATCH 07/17] Use argparse module in mapsme_json_to_cities.py --- mapsme_json_to_cities.py | 45 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/mapsme_json_to_cities.py b/mapsme_json_to_cities.py index cac7946..2449f0b 100644 --- a/mapsme_json_to_cities.py +++ b/mapsme_json_to_cities.py @@ -1,30 +1,41 @@ -""" -This script generates a list of good/all network names. -It is used by subway render to generate the list of network at frontend. -It uses two sources: a mapsme.json validator output with good networks, and -a google spreadsheet with networks for the process_subways.download_cities() -function. -""" - +import argparse import json -import sys from process_subways import download_cities if __name__ == '__main__': - if len(sys.argv) < 2: - print('Usage: {} [--with-bad]'.format(sys.argv[0])) - sys.exit(1) + arg_parser = argparse.ArgumentParser( + description=""" + This script generates a list of good/all network names. + It is used by subway render to generate the list of network at frontend. + It uses two sources: a mapsme.json validator output with good networks, and + a google spreadsheet with networks for the process_subways.download_cities() + function.""", + formatter_class=argparse.RawTextHelpFormatter, + ) - with_bad = len(sys.argv) > 2 and sys.argv[2] == '--with-bad' + arg_parser.add_argument( + 'subway_json_file', + type=argparse.FileType('r'), + help="Validator output defined by -o option of process_subways.py script" + ) + + arg_parser.add_argument( + '--with-bad', + action="store_true", + help="Whether to include cities validation of which was failed", + ) + + args = arg_parser.parse_args() + + with_bad = args.with_bad + subway_json_file = args.subway_json_file + subway_json = json.load(subway_json_file) - subway_json_file = sys.argv[1] - with open(subway_json_file) as f: - subway_json = json.load(f) good_cities = set(n.get('network', n.get('title')) for n in subway_json['networks']) - cities = download_cities() + lines = [] for c in cities: if c.name in good_cities: -- 2.45.3 From 6596d9789cb280d9cc7ff1ac39cecf4d0317c4d1 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Sun, 27 Jun 2021 00:50:37 +0300 Subject: [PATCH 08/17] PEPify-8 with black -l 79 -S --- make_all_metro_poly.py | 14 +- mapsme_json_to_cities.py | 6 +- process_subways.py | 206 +++++--- processors/mapsme.py | 183 ++++--- scripts/process_subways.sh | 2 +- stop_areas/make_stop_areas.py | 85 +++- stop_areas/make_tram_areas.py | 68 ++- stop_areas/serve.py | 5 +- subway_io.py | 180 ++++--- subway_structure.py | 919 ++++++++++++++++++++++++---------- validation_to_html.py | 136 +++-- 11 files changed, 1273 insertions(+), 531 deletions(-) diff --git a/make_all_metro_poly.py b/make_all_metro_poly.py index 4276b2b..610892d 100644 --- a/make_all_metro_poly.py +++ b/make_all_metro_poly.py @@ -9,12 +9,14 @@ def make_disjoint_metro_polygons(): polygons = [] for c in cities: - polygon = shapely.geometry.Polygon([ - (c.bbox[1], c.bbox[0]), - (c.bbox[1], c.bbox[2]), - (c.bbox[3], c.bbox[2]), - (c.bbox[3], c.bbox[0]), - ]) + polygon = shapely.geometry.Polygon( + [ + (c.bbox[1], c.bbox[0]), + (c.bbox[1], c.bbox[2]), + (c.bbox[3], c.bbox[2]), + (c.bbox[3], c.bbox[0]), + ] + ) polygons.append(polygon) union = shapely.ops.unary_union(polygons) diff --git a/mapsme_json_to_cities.py b/mapsme_json_to_cities.py index 2449f0b..4b8fea8 100644 --- a/mapsme_json_to_cities.py +++ b/mapsme_json_to_cities.py @@ -18,7 +18,7 @@ if __name__ == '__main__': arg_parser.add_argument( 'subway_json_file', type=argparse.FileType('r'), - help="Validator output defined by -o option of process_subways.py script" + help="Validator output defined by -o option of process_subways.py script", ) arg_parser.add_argument( @@ -33,7 +33,9 @@ if __name__ == '__main__': subway_json_file = args.subway_json_file subway_json = json.load(subway_json_file) - good_cities = set(n.get('network', n.get('title')) for n in subway_json['networks']) + good_cities = set( + n.get('network', n.get('title')) for n in subway_json['networks'] + ) cities = download_cities() lines = [] diff --git a/process_subways.py b/process_subways.py index 806b150..1a25207 100755 --- a/process_subways.py +++ b/process_subways.py @@ -39,13 +39,17 @@ def overpass_request(overground, overpass_api, bboxes): if not overground: query += 'node[railway=subway_entrance]{};'.format(bbox_part) query += 'rel[public_transport=stop_area]{};'.format(bbox_part) - query += 'rel(br)[type=public_transport][public_transport=stop_area_group];' + query += ( + 'rel(br)[type=public_transport][public_transport=stop_area_group];' + ) query += ');(._;>>;);out body center qt;' logging.debug('Query: %s', query) url = '{}?data={}'.format(overpass_api, urllib.parse.quote(query)) response = urllib.request.urlopen(url, timeout=1000) if response.getcode() != 200: - raise Exception('Failed to query Overpass API: HTTP {}'.format(response.getcode())) + raise Exception( + 'Failed to query Overpass API: HTTP {}'.format(response.getcode()) + ) return json.load(response)['elements'] @@ -56,7 +60,11 @@ def multi_overpass(overground, overpass_api, bboxes): for i in range(0, len(bboxes) + SLICE_SIZE - 1, SLICE_SIZE): if i > 0: time.sleep(INTERREQUEST_WAIT) - result.extend(overpass_request(overground, overpass_api, bboxes[i:i+SLICE_SIZE])) + result.extend( + overpass_request( + overground, overpass_api, bboxes[i : i + SLICE_SIZE] + ) + ) return result @@ -66,14 +74,14 @@ def slugify(name): def calculate_centers(elements): """Adds 'center' key to each way/relation in elements, - except for empty ways or relations. - Relies on nodes-ways-relations order in the elements list. + except for empty ways or relations. + Relies on nodes-ways-relations order in the elements list. """ - nodes = {} # id(int) => (lat, lon) - ways = {} # id(int) => (lat, lon) + nodes = {} # id(int) => (lat, lon) + ways = {} # id(int) => (lat, lon) relations = {} # id(int) => (lat, lon) empty_relations = set() # ids(int) of relations without members - # or containing only empty relations + # or containing only empty relations def calculate_way_center(el): # If element has been queried via overpass-api with 'out center;' @@ -108,9 +116,13 @@ def calculate_centers(elements): else: # Center of child relation is not known yet return False - member_container = (nodes if m['type'] == 'node' else - ways if m['type'] == 'way' else - relations) + member_container = ( + nodes + if m['type'] == 'node' + else ways + if m['type'] == 'way' + else relations + ) if m['ref'] in member_container: center[0] += member_container[m['ref']][0] center[1] += member_container[m['ref']][1] @@ -145,54 +157,104 @@ def calculate_centers(elements): relations_without_center = new_relations_without_center if relations_without_center: - logging.error("Cannot calculate center for the relations (%d in total): %s%s", - len(relations_without_center), - ', '.join(str(rel['id']) for rel in relations_without_center[:20]), - ", ..." if len(relations_without_center) > 20 else "") + logging.error( + "Cannot calculate center for the relations (%d in total): %s%s", + len(relations_without_center), + ', '.join(str(rel['id']) for rel in relations_without_center[:20]), + ", ..." if len(relations_without_center) > 20 else "", + ) if empty_relations: - logging.warning("Empty relations (%d in total): %s%s", - len(empty_relations), - ', '.join(str(x) for x in list(empty_relations)[:20]), - ", ..." if len(empty_relations) > 20 else "") + logging.warning( + "Empty relations (%d in total): %s%s", + len(empty_relations), + ', '.join(str(x) for x in list(empty_relations)[:20]), + ", ..." if len(empty_relations) > 20 else "", + ) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '-i', '--source', help='File to write backup of OSM data, or to read data from') - parser.add_argument('-x', '--xml', help='OSM extract with routes, to read data from') - parser.add_argument('--overpass-api', - default='http://overpass-api.de/api/interpreter', - help="Overpass API URL") - parser.add_argument('-q', '--quiet', action='store_true', help='Show only warnings and errors') - parser.add_argument('-c', '--city', help='Validate only a single city or a country') - parser.add_argument('-t', '--overground', action='store_true', - help='Process overground transport instead of subways') - parser.add_argument('-e', '--entrances', type=argparse.FileType('w', encoding='utf-8'), - help='Export unused subway entrances as GeoJSON here') - parser.add_argument('-l', '--log', type=argparse.FileType('w', encoding='utf-8'), - help='Validation JSON file name') - parser.add_argument('-o', '--output', type=argparse.FileType('w', encoding='utf-8'), - help='Processed metro systems output') + '-i', + '--source', + help='File to write backup of OSM data, or to read data from', + ) + parser.add_argument( + '-x', '--xml', help='OSM extract with routes, to read data from' + ) + parser.add_argument( + '--overpass-api', + default='http://overpass-api.de/api/interpreter', + help="Overpass API URL", + ) + parser.add_argument( + '-q', + '--quiet', + action='store_true', + help='Show only warnings and errors', + ) + parser.add_argument( + '-c', '--city', help='Validate only a single city or a country' + ) + parser.add_argument( + '-t', + '--overground', + action='store_true', + help='Process overground transport instead of subways', + ) + parser.add_argument( + '-e', + '--entrances', + type=argparse.FileType('w', encoding='utf-8'), + help='Export unused subway entrances as GeoJSON here', + ) + parser.add_argument( + '-l', + '--log', + type=argparse.FileType('w', encoding='utf-8'), + help='Validation JSON file name', + ) + parser.add_argument( + '-o', + '--output', + type=argparse.FileType('w', encoding='utf-8'), + help='Processed metro systems output', + ) parser.add_argument('--cache', help='Cache file name for processed data') - parser.add_argument('-r', '--recovery-path', help='Cache file name for error recovery') - parser.add_argument('-d', '--dump', help='Make a YAML file for a city data') - parser.add_argument('-j', '--geojson', help='Make a GeoJSON file for a city data') - parser.add_argument('--crude', action='store_true', - help='Do not use OSM railway geometry for GeoJSON') + parser.add_argument( + '-r', '--recovery-path', help='Cache file name for error recovery' + ) + parser.add_argument( + '-d', '--dump', help='Make a YAML file for a city data' + ) + parser.add_argument( + '-j', '--geojson', help='Make a GeoJSON file for a city data' + ) + parser.add_argument( + '--crude', + action='store_true', + help='Do not use OSM railway geometry for GeoJSON', + ) options = parser.parse_args() if options.quiet: log_level = logging.WARNING else: log_level = logging.INFO - logging.basicConfig(level=log_level, datefmt='%H:%M:%S', - format='%(asctime)s %(levelname)-7s %(message)s') + logging.basicConfig( + level=log_level, + datefmt='%H:%M:%S', + format='%(asctime)s %(levelname)-7s %(message)s', + ) # Downloading cities from Google Spreadsheets cities = download_cities(options.overground) if options.city: - cities = [c for c in cities if c.name == options.city or c.country == options.city] + cities = [ + c + for c in cities + if c.name == options.city or c.country == options.city + ] if not cities: logging.error('No cities to process') sys.exit(2) @@ -223,8 +285,10 @@ if __name__ == '__main__': json.dump(osm, f) else: if len(cities) > 10: - logging.error('Would not download that many cities from Overpass API, ' - 'choose a smaller set') + logging.error( + 'Would not download that many cities from Overpass API, ' + 'choose a smaller set' + ) sys.exit(3) bboxes = [c.bbox for c in cities] logging.info('Downloading data from Overpass API') @@ -247,10 +311,18 @@ if __name__ == '__main__': try: c.extract_routes() except CriticalValidationError as e: - logging.error("Critical validation error while processing %s: %s", c.name, str(e)) + logging.error( + "Critical validation error while processing %s: %s", + c.name, + str(e), + ) c.error(str(e)) except AssertionError as e: - logging.error("Validation logic error while processing %s: %s", c.name, str(e)) + logging.error( + "Validation logic error while processing %s: %s", + c.name, + str(e), + ) c.error("Validation logic error: {}".format(str(e))) else: c.validate() @@ -261,11 +333,17 @@ if __name__ == '__main__': transfers = find_transfers(osm, cities) good_city_names = set(c.name for c in good_cities) - logging.info('%s good cities: %s', len(good_city_names), - ', '.join(sorted(good_city_names))) + logging.info( + '%s good cities: %s', + len(good_city_names), + ', '.join(sorted(good_city_names)), + ) bad_city_names = set(c.name for c in cities) - good_city_names - logging.info('%s bad cities: %s', len(bad_city_names), - ', '.join(sorted(bad_city_names))) + logging.info( + '%s bad cities: %s', + len(bad_city_names), + ', '.join(sorted(bad_city_names)), + ) if options.recovery_path: write_recovery_data(options.recovery_path, recovery_data, cities) @@ -276,8 +354,11 @@ if __name__ == '__main__': if options.dump: if os.path.isdir(options.dump): for c in cities: - with open(os.path.join(options.dump, slugify(c.name) + '.yaml'), - 'w', encoding='utf-8') as f: + with open( + os.path.join(options.dump, slugify(c.name) + '.yaml'), + 'w', + encoding='utf-8', + ) as f: dump_yaml(c, f) elif len(cities) == 1: with open(options.dump, 'w', encoding='utf-8') as f: @@ -288,14 +369,21 @@ if __name__ == '__main__': if options.geojson: if os.path.isdir(options.geojson): for c in cities: - with open(os.path.join(options.geojson, slugify(c.name) + '.geojson'), - 'w', encoding='utf-8') as f: + with open( + os.path.join( + options.geojson, slugify(c.name) + '.geojson' + ), + 'w', + encoding='utf-8', + ) as f: json.dump(make_geojson(c, not options.crude), f) elif len(cities) == 1: with open(options.geojson, 'w', encoding='utf-8') as f: json.dump(make_geojson(cities[0], not options.crude), f) else: - logging.error('Cannot make a geojson of %s cities at once', len(cities)) + logging.error( + 'Cannot make a geojson of %s cities at once', len(cities) + ) if options.log: res = [] @@ -306,5 +394,9 @@ if __name__ == '__main__': json.dump(res, options.log, indent=2, ensure_ascii=False) if options.output: - json.dump(processor.process(cities, transfers, options.cache), - options.output, indent=1, ensure_ascii=False) + json.dump( + processor.process(cities, transfers, options.cache), + options.output, + indent=1, + ensure_ascii=False, + ) diff --git a/processors/mapsme.py b/processors/mapsme.py index 9855b27..c33c97f 100755 --- a/processors/mapsme.py +++ b/processors/mapsme.py @@ -3,15 +3,17 @@ import os import logging from collections import defaultdict from subway_structure import ( - distance, el_center, Station, - DISPLACEMENT_TOLERANCE + distance, + el_center, + Station, + DISPLACEMENT_TOLERANCE, ) OSM_TYPES = {'n': (0, 'node'), 'w': (2, 'way'), 'r': (3, 'relation')} ENTRANCE_PENALTY = 60 # seconds TRANSFER_PENALTY = 30 # seconds -KMPH_TO_MPS = 1/3.6 # km/h to m/s conversion multiplier +KMPH_TO_MPS = 1 / 3.6 # km/h to m/s conversion multiplier SPEED_TO_ENTRANCE = 5 * KMPH_TO_MPS # m/s SPEED_ON_TRANSFER = 3.5 * KMPH_TO_MPS # m/s SPEED_ON_LINE = 40 * KMPH_TO_MPS # m/s @@ -37,18 +39,22 @@ class DummyCache: def __getattr__(self, name): """This results in that a call to any method effectively does nothing and does not generate exceptions.""" + def method(*args, **kwargs): return None + return method def if_object_is_used(method): """Decorator to skip method execution under certain condition. Relies on "is_used" object property.""" + def inner(self, *args, **kwargs): if not self.is_used: return return method(self, *args, **kwargs) + return inner @@ -66,8 +72,11 @@ class MapsmeCache: with open(cache_path, 'r', encoding='utf-8') as f: self.cache = json.load(f) except json.decoder.JSONDecodeError: - logging.warning("City cache '%s' is not a valid json file. " - "Building cache from scratch.", cache_path) + logging.warning( + "City cache '%s' is not a valid json file. " + "Building cache from scratch.", + cache_path, + ) self.recovered_city_names = set() # One stoparea may participate in routes of different cities self.stop_cities = defaultdict(set) # stoparea id -> city names @@ -80,15 +89,20 @@ class MapsmeCache: """ city_cache_data = self.cache[city.name] for stoparea_id, cached_stoparea in city_cache_data['stops'].items(): - station_id = cached_stoparea['osm_type'][0] + str(cached_stoparea['osm_id']) + station_id = cached_stoparea['osm_type'][0] + str( + cached_stoparea['osm_id'] + ) city_station = city.elements.get(station_id) - if (not city_station or - not Station.is_station(city_station, city.modes)): + if not city_station or not Station.is_station( + city_station, city.modes + ): return False station_coords = el_center(city_station) - cached_station_coords = tuple(cached_stoparea[coord] for coord in ('lon', 'lat')) + cached_station_coords = tuple( + cached_stoparea[coord] for coord in ('lon', 'lat') + ) displacement = distance(station_coords, cached_station_coords) - if displacement > DISPLACEMENT_TOLERANCE: + if displacement > DISPLACEMENT_TOLERANCE: return False return True @@ -123,7 +137,7 @@ class MapsmeCache: self.cache[city_name] = { 'network': network, 'stops': {}, # stoparea el_id -> jsonified stop data - 'transfers': [] # list of tuples (stoparea1_uid, stoparea2_uid, time); uid1 < uid2 + 'transfers': [], # list of tuples (stoparea1_uid, stoparea2_uid, time); uid1 < uid2 } @if_object_is_used @@ -142,9 +156,11 @@ class MapsmeCache: @if_object_is_used def add_transfer(self, stoparea1_uid, stoparea2_uid, transfer_time): """If a transfer is inside a good city, add it to the city's cache.""" - for city_name in (self.good_city_names & - self.stop_cities[stoparea1_uid] & - self.stop_cities[stoparea2_uid]): + for city_name in ( + self.good_city_names + & self.stop_cities[stoparea1_uid] + & self.stop_cities[stoparea2_uid] + ): self.cache[city_name]['transfers'].append( (stoparea1_uid, stoparea2_uid, transfer_time) ) @@ -186,7 +202,6 @@ def process(cities, transfers, cache_path): exits.append(n) return exits - cache = MapsmeCache(cache_path, cities) stop_areas = {} # stoparea el_id -> StopArea instance @@ -206,7 +221,7 @@ def process(cities, transfers, cache_path): 'name': route.name, 'colour': format_colour(route.colour), 'route_id': uid(route.id, 'r'), - 'itineraries': [] + 'itineraries': [], } if route.infill: routes['casing'] = routes['colour'] @@ -216,33 +231,62 @@ def process(cities, transfers, cache_path): for stop in variant: stop_areas[stop.stoparea.id] = stop.stoparea cache.link_stop_with_city(stop.stoparea.id, city.name) - itin.append([uid(stop.stoparea.id), round(stop.distance/SPEED_ON_LINE)]) + itin.append( + [ + uid(stop.stoparea.id), + round(stop.distance / SPEED_ON_LINE), + ] + ) # Make exits from platform nodes, if we don't have proper exits - if len(stop.stoparea.entrances) + len(stop.stoparea.exits) == 0: + if ( + len(stop.stoparea.entrances) + len(stop.stoparea.exits) + == 0 + ): for pl in stop.stoparea.platforms: pl_el = city.elements[pl] if pl_el['type'] == 'node': pl_nodes = [pl_el] elif pl_el['type'] == 'way': - pl_nodes = [city.elements.get('n{}'.format(n)) - for n in pl_el['nodes']] + pl_nodes = [ + city.elements.get('n{}'.format(n)) + for n in pl_el['nodes'] + ] else: pl_nodes = [] for m in pl_el['members']: if m['type'] == 'way': - if '{}{}'.format(m['type'][0], m['ref']) in city.elements: + if ( + '{}{}'.format( + m['type'][0], m['ref'] + ) + in city.elements + ): pl_nodes.extend( - [city.elements.get('n{}'.format(n)) - for n in city.elements['{}{}'.format( - m['type'][0], m['ref'])]['nodes']]) + [ + city.elements.get( + 'n{}'.format(n) + ) + for n in city.elements[ + '{}{}'.format( + m['type'][0], + m['ref'], + ) + ]['nodes'] + ] + ) pl_nodes = [n for n in pl_nodes if n] platform_nodes[pl] = find_exits_for_platform( - stop.stoparea.centers[pl], pl_nodes) + stop.stoparea.centers[pl], pl_nodes + ) - routes['itineraries'].append({ - 'stops': itin, - 'interval': round((variant.interval or DEFAULT_INTERVAL) * 60) - }) + routes['itineraries'].append( + { + 'stops': itin, + 'interval': round( + (variant.interval or DEFAULT_INTERVAL) * 60 + ), + } + ) network['routes'].append(routes) networks.append(network) @@ -261,41 +305,57 @@ def process(cities, transfers, cache_path): for e_l, k in ((stop.entrances, 'entrances'), (stop.exits, 'exits')): for e in e_l: if e[0] == 'n': - st[k].append({ - 'osm_type': 'node', - 'osm_id': int(e[1:]), - 'lon': stop.centers[e][0], - 'lat': stop.centers[e][1], - 'distance': ENTRANCE_PENALTY + round(distance( - stop.centers[e], stop.center)/SPEED_TO_ENTRANCE) - }) + st[k].append( + { + 'osm_type': 'node', + 'osm_id': int(e[1:]), + 'lon': stop.centers[e][0], + 'lat': stop.centers[e][1], + 'distance': ENTRANCE_PENALTY + + round( + distance(stop.centers[e], stop.center) + / SPEED_TO_ENTRANCE + ), + } + ) if len(stop.entrances) + len(stop.exits) == 0: if stop.platforms: for pl in stop.platforms: for n in platform_nodes[pl]: for k in ('entrances', 'exits'): - st[k].append({ - 'osm_type': n['type'], - 'osm_id': n['id'], - 'lon': n['lon'], - 'lat': n['lat'], - 'distance': ENTRANCE_PENALTY + round(distance( - (n['lon'], n['lat']), stop.center)/SPEED_TO_ENTRANCE) - }) + st[k].append( + { + 'osm_type': n['type'], + 'osm_id': n['id'], + 'lon': n['lon'], + 'lat': n['lat'], + 'distance': ENTRANCE_PENALTY + + round( + distance( + (n['lon'], n['lat']), stop.center + ) + / SPEED_TO_ENTRANCE + ), + } + ) else: for k in ('entrances', 'exits'): - st[k].append({ - 'osm_type': OSM_TYPES[stop.station.id[0]][1], - 'osm_id': int(stop.station.id[1:]), - 'lon': stop.centers[stop.id][0], - 'lat': stop.centers[stop.id][1], - 'distance': 60 - }) + st[k].append( + { + 'osm_type': OSM_TYPES[stop.station.id[0]][1], + 'osm_id': int(stop.station.id[1:]), + 'lon': stop.centers[stop.id][0], + 'lat': stop.centers[stop.id][1], + 'distance': 60, + } + ) stops[stop_id] = st cache.add_stop(stop_id, st) - pairwise_transfers = {} # (stoparea1_uid, stoparea2_uid) -> time; uid1 < uid2 + pairwise_transfers = ( + {} + ) # (stoparea1_uid, stoparea2_uid) -> time; uid1 < uid2 for t_set in transfers: t = list(t_set) for t_first in range(len(t) - 1): @@ -306,23 +366,24 @@ def process(cities, transfers, cache_path): uid1 = uid(stoparea1.id) uid2 = uid(stoparea2.id) uid1, uid2 = sorted([uid1, uid2]) - transfer_time = (TRANSFER_PENALTY - + round(distance(stoparea1.center, - stoparea2.center) - / SPEED_ON_TRANSFER)) + transfer_time = TRANSFER_PENALTY + round( + distance(stoparea1.center, stoparea2.center) + / SPEED_ON_TRANSFER + ) pairwise_transfers[(uid1, uid2)] = transfer_time cache.add_transfer(uid1, uid2, transfer_time) cache.provide_transfers(pairwise_transfers) cache.save() - pairwise_transfers = [(stop1_uid, stop2_uid, transfer_time) - for (stop1_uid, stop2_uid), transfer_time - in pairwise_transfers.items()] + pairwise_transfers = [ + (stop1_uid, stop2_uid, transfer_time) + for (stop1_uid, stop2_uid), transfer_time in pairwise_transfers.items() + ] result = { 'stops': list(stops.values()), 'transfers': pairwise_transfers, - 'networks': networks + 'networks': networks, } return result diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 8a02442..b0d5f3b 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -19,7 +19,7 @@ In more detail, the script does the following: - Copies results onto remote server, if it is set up. During this procedure, as many steps are skipped as possible. Namely: - - Making metro extract is skipped if \$PLANET_METRO variable is set and the file exists. + - Generation of metro extract is skipped if \$PLANET_METRO variable is set and the file exists. - Update with osmupdate is skipped if \$SKIP_PLANET_UPDATE or \$SKIP_FILTERING is set. - Filtering is skipped if \$SKIP_FILTERING is set and \$FILTERED_DATA is set and the file exists. diff --git a/stop_areas/make_stop_areas.py b/stop_areas/make_stop_areas.py index 9f059b9..43699a9 100755 --- a/stop_areas/make_stop_areas.py +++ b/stop_areas/make_stop_areas.py @@ -54,17 +54,21 @@ class StationWrapper: def distance(self, other): """Calculate distance in meters.""" dx = math.radians(self[0] - other['lon']) * math.cos( - 0.5 * math.radians(self[1] + other['lat'])) + 0.5 * math.radians(self[1] + other['lat']) + ) dy = math.radians(self[1] - other['lat']) - return 6378137 * math.sqrt(dx*dx + dy*dy) + return 6378137 * math.sqrt(dx * dx + dy * dy) def overpass_request(bbox): url = 'http://overpass-api.de/api/interpreter?data={}'.format( - urllib.parse.quote(QUERY.replace('{{bbox}}', bbox))) + urllib.parse.quote(QUERY.replace('{{bbox}}', bbox)) + ) response = urllib.request.urlopen(url, timeout=1000) if response.getcode() != 200: - raise Exception('Failed to query Overpass API: HTTP {}'.format(response.getcode())) + raise Exception( + 'Failed to query Overpass API: HTTP {}'.format(response.getcode()) + ) reader = codecs.getreader('utf-8') return json.load(reader(response))['elements'] @@ -80,8 +84,11 @@ def add_stop_areas(src): stations[el_id(el)] = el for el in src: - if (el['type'] == 'relation' and 'tags' in el and - el['tags'].get('route', None) in ('subway', 'light_rail')): + if ( + el['type'] == 'relation' + and 'tags' in el + and el['tags'].get('route', None) in ('subway', 'light_rail') + ): for m in el['members']: st = stations.get(el_id(m), None) if st and 'station' not in st['tags']: @@ -91,7 +98,10 @@ def add_stop_areas(src): # Create a kd-tree out of subway stations stations = kdtree.create(dimensions=2) for el in src: - if 'tags' in el and el['tags'].get('station', None) in ('subway', 'light_rail'): + if 'tags' in el and el['tags'].get('station', None) in ( + 'subway', + 'light_rail', + ): stations.add(StationWrapper(el)) if stations.is_leaf: @@ -105,13 +115,21 @@ def add_stop_areas(src): continue if 'station' in el['tags']: continue - if (el['tags'].get('railway', None) not in ('subway_entrance', 'platform') and - el['tags'].get('public_transport', None) not in ('platform', 'stop_position')): + if el['tags'].get('railway', None) not in ( + 'subway_entrance', + 'platform', + ) and el['tags'].get('public_transport', None) not in ( + 'platform', + 'stop_position', + ): continue coords = el.get('center', el) station = stations.search_nn((coords['lon'], coords['lat']))[0].data if station.distance(coords) < MAX_DISTANCE: - k = (station.station['id'], station.station['tags'].get('name', 'station_with_no_name')) + k = ( + station.station['id'], + station.station['tags'].get('name', 'station_with_no_name'), + ) # Disregard exits and platforms that are differently named if el['tags'].get('name', k[1]) == k[1]: if k not in stop_areas: @@ -120,7 +138,10 @@ def add_stop_areas(src): # Find existing stop_area relations for stations and remove these stations for el in src: - if el['type'] == 'relation' and el['tags'].get('public_transport', None) == 'stop_area': + if ( + el['type'] == 'relation' + and el['tags'].get('public_transport', None) == 'stop_area' + ): found = False for m in el['members']: if found: @@ -141,18 +162,35 @@ def add_stop_areas(src): etree.SubElement(rel, 'tag', k='public_transport', v='stop_area') etree.SubElement(rel, 'tag', k='name', v=st[1]) for m in members.values(): - if m['tags'].get('railway', m['tags'].get('public_transport', None)) == 'platform': + if ( + m['tags'].get( + 'railway', m['tags'].get('public_transport', None) + ) + == 'platform' + ): role = 'platform' elif m['tags'].get('public_transport', None) == 'stop_position': role = 'stop' else: role = '' - etree.SubElement(rel, 'member', ref=str(m['id']), type=m['type'], role=role) + etree.SubElement( + rel, 'member', ref=str(m['id']), type=m['type'], role=role + ) # Add all downloaded elements for el in src: obj = etree.SubElement(root, el['type']) - for a in ('id', 'type', 'user', 'uid', 'version', 'changeset', 'timestamp', 'lat', 'lon'): + for a in ( + 'id', + 'type', + 'user', + 'uid', + 'version', + 'changeset', + 'timestamp', + 'lat', + 'lon', + ): if a in el: obj.set(a, str(el[a])) if 'modified' in el: @@ -162,8 +200,13 @@ def add_stop_areas(src): etree.SubElement(obj, 'tag', k=k, v=v) if 'members' in el: for m in el['members']: - etree.SubElement(obj, 'member', ref=str(m['ref']), - type=m['type'], role=m.get('role', '')) + etree.SubElement( + obj, + 'member', + ref=str(m['ref']), + type=m['type'], + role=m.get('role', ''), + ) if 'nodes' in el: for n in el['nodes']: etree.SubElement(obj, 'nd', ref=str(n)) @@ -173,8 +216,14 @@ def add_stop_areas(src): if __name__ == '__main__': if len(sys.argv) < 2: - print('Read a JSON from Overpass and output JOSM OSM XML with added stop_area relations') - print('Usage: {} {{|}} [output.osm]'.format(sys.argv[0])) + print( + 'Read a JSON from Overpass and output JOSM OSM XML with added stop_area relations' + ) + print( + 'Usage: {} {{|}} [output.osm]'.format( + sys.argv[0] + ) + ) sys.exit(1) if re.match(r'^[-0-9.,]+$', sys.argv[1]): diff --git a/stop_areas/make_tram_areas.py b/stop_areas/make_tram_areas.py index 443faba..f06fdac 100755 --- a/stop_areas/make_tram_areas.py +++ b/stop_areas/make_tram_areas.py @@ -45,17 +45,21 @@ class StationWrapper: def distance(self, other): """Calculate distance in meters.""" dx = math.radians(self[0] - other['lon']) * math.cos( - 0.5 * math.radians(self[1] + other['lat'])) + 0.5 * math.radians(self[1] + other['lat']) + ) dy = math.radians(self[1] - other['lat']) - return 6378137 * math.sqrt(dx*dx + dy*dy) + return 6378137 * math.sqrt(dx * dx + dy * dy) def overpass_request(bbox): url = 'http://overpass-api.de/api/interpreter?data={}'.format( - urllib.parse.quote(QUERY.replace('{{bbox}}', bbox))) + urllib.parse.quote(QUERY.replace('{{bbox}}', bbox)) + ) response = urllib.request.urlopen(url, timeout=1000) if response.getcode() != 200: - raise Exception('Failed to query Overpass API: HTTP {}'.format(response.getcode())) + raise Exception( + 'Failed to query Overpass API: HTTP {}'.format(response.getcode()) + ) reader = codecs.getreader('utf-8') return json.load(reader(response))['elements'] @@ -91,7 +95,11 @@ def add_stop_areas(src): stop_areas = {} for el in src: # Only tram routes - if 'tags' not in el or el['type'] != 'relation' or el['tags'].get('route') != 'tram': + if ( + 'tags' not in el + or el['type'] != 'relation' + or el['tags'].get('route') != 'tram' + ): continue for m in el['members']: if el_id(m) not in elements: @@ -102,16 +110,24 @@ def add_stop_areas(src): if pel['tags'].get('railway') == 'tram_stop': continue coords = pel.get('center', pel) - station = stations.search_nn((coords['lon'], coords['lat']))[0].data + station = stations.search_nn( + (coords['lon'], coords['lat']) + )[0].data if station.distance(coords) < MAX_DISTANCE: - k = (station.station['id'], station.station['tags'].get('name', None)) + k = ( + station.station['id'], + station.station['tags'].get('name', None), + ) if k not in stop_areas: stop_areas[k] = {el_id(station.station): station.station} stop_areas[k][el_id(m)] = pel # Find existing stop_area relations for stations and remove these stations for el in src: - if el['type'] == 'relation' and el['tags'].get('public_transport', None) == 'stop_area': + if ( + el['type'] == 'relation' + and el['tags'].get('public_transport', None) == 'stop_area' + ): found = False for m in el['members']: if found: @@ -133,12 +149,24 @@ def add_stop_areas(src): if st[1]: etree.SubElement(rel, 'tag', k='name', v=st[1]) for m in members.values(): - etree.SubElement(rel, 'member', ref=str(m['id']), type=m['type'], role='') + etree.SubElement( + rel, 'member', ref=str(m['id']), type=m['type'], role='' + ) # Add all downloaded elements for el in src: obj = etree.SubElement(root, el['type']) - for a in ('id', 'type', 'user', 'uid', 'version', 'changeset', 'timestamp', 'lat', 'lon'): + for a in ( + 'id', + 'type', + 'user', + 'uid', + 'version', + 'changeset', + 'timestamp', + 'lat', + 'lon', + ): if a in el: obj.set(a, str(el[a])) if 'modified' in el: @@ -148,8 +176,13 @@ def add_stop_areas(src): etree.SubElement(obj, 'tag', k=k, v=v) if 'members' in el: for m in el['members']: - etree.SubElement(obj, 'member', ref=str(m['ref']), - type=m['type'], role=m.get('role', '')) + etree.SubElement( + obj, + 'member', + ref=str(m['ref']), + type=m['type'], + role=m.get('role', ''), + ) if 'nodes' in el: for n in el['nodes']: etree.SubElement(obj, 'nd', ref=str(n)) @@ -159,8 +192,15 @@ def add_stop_areas(src): if __name__ == '__main__': if len(sys.argv) < 2: - print('Read a JSON from Overpass and output JOSM OSM XML with added stop_area relations') - print('Usage: {} {{|}} [output.osm]'.format(sys.argv[0])) + print( + 'Read a JSON from Overpass and output JOSM OSM XML ' + 'with added stop_area relations' + ) + print( + 'Usage: {} {{|}} [output.osm]'.format( + sys.argv[0] + ) + ) sys.exit(1) if re.match(r'^[-0-9.,]+$', sys.argv[1]): diff --git a/stop_areas/serve.py b/stop_areas/serve.py index c6f4c49..e5d695e 100755 --- a/stop_areas/serve.py +++ b/stop_areas/serve.py @@ -18,8 +18,11 @@ def convert(): return 'No data from overpass, sorry.' result = add_stop_areas(src) response = make_response(result) - response.headers['Content-Disposition'] = 'attachment; filename="stop_areas.osm"' + response.headers['Content-Disposition'] = ( + 'attachment; filename="stop_areas.osm"' + ) return response + if __name__ == '__main__': app.run() diff --git a/subway_io.py b/subway_io.py index abf1aee..810c02e 100644 --- a/subway_io.py +++ b/subway_io.py @@ -26,9 +26,13 @@ def load_xml(f): elif sub.tag == 'nd': nd.append(int(sub.get('ref'))) elif sub.tag == 'member': - members.append({'type': sub.get('type'), - 'ref': int(sub.get('ref')), - 'role': sub.get('role', '')}) + members.append( + { + 'type': sub.get('type'), + 'ref': int(sub.get('ref')), + 'role': sub.get('role', ''), + } + ) if tags: el['tags'] = tags if nd: @@ -44,13 +48,15 @@ def load_xml(f): _YAML_SPECIAL_CHARACTERS = "!&*{}[],#|>@`'\"" _YAML_SPECIAL_SEQUENCES = ("- ", ": ", "? ") + def _get_yaml_compatible_string(scalar): """Enclose string in single quotes in some cases""" string = str(scalar) - if (string and - (string[0] in _YAML_SPECIAL_CHARACTERS - or any(seq in string for seq in _YAML_SPECIAL_SEQUENCES) - or string.endswith(':'))): + if string and ( + string[0] in _YAML_SPECIAL_CHARACTERS + or any(seq in string for seq in _YAML_SPECIAL_SEQUENCES) + or string.endswith(':') + ): string = string.replace("'", "''") string = "'{}'".format(string) return string @@ -81,7 +87,9 @@ def dump_yaml(city, f): stops = set() routes = [] for route in city: - stations = OrderedDict([(sa.transfer or sa.id, sa.name) for sa in route.stop_areas()]) + stations = OrderedDict( + [(sa.transfer or sa.id, sa.name) for sa in route.stop_areas()] + ) rte = { 'type': route.mode, 'ref': route.ref, @@ -90,7 +98,7 @@ def dump_yaml(city, f): 'infill': route.infill, 'station_count': len(stations), 'stations': list(stations.values()), - 'itineraries': {} + 'itineraries': {}, } for variant in route: if INCLUDE_STOP_AREAS: @@ -98,14 +106,22 @@ def dump_yaml(city, f): for st in variant: s = st.stoparea if s.id == s.station.id: - v_stops.append('{} ({})'.format(s.station.name, s.station.id)) + v_stops.append( + '{} ({})'.format(s.station.name, s.station.id) + ) else: - v_stops.append('{} ({}) in {} ({})'.format(s.station.name, s.station.id, - s.name, s.id)) + v_stops.append( + '{} ({}) in {} ({})'.format( + s.station.name, s.station.id, s.name, s.id + ) + ) else: - v_stops = ['{} ({})'.format( - s.stoparea.station.name, - s.stoparea.station.id) for s in variant] + v_stops = [ + '{} ({})'.format( + s.stoparea.station.name, s.stoparea.station.id + ) + for s in variant + ] rte['itineraries'][variant.id] = v_stops stops.update(v_stops) routes.append(rte) @@ -132,64 +148,73 @@ def make_geojson(city, tracks=True): for rmaster in city: for variant in rmaster: if not tracks: - features.append({ - 'type': 'Feature', - 'geometry': { - 'type': 'LineString', - 'coordinates': [s.stop for s in variant], - }, - 'properties': { - 'ref': variant.ref, - 'name': variant.name, - 'stroke': variant.colour + features.append( + { + 'type': 'Feature', + 'geometry': { + 'type': 'LineString', + 'coordinates': [s.stop for s in variant], + }, + 'properties': { + 'ref': variant.ref, + 'name': variant.name, + 'stroke': variant.colour, + }, } - }) + ) elif variant.tracks: - features.append({ - 'type': 'Feature', - 'geometry': { - 'type': 'LineString', - 'coordinates': variant.tracks, - }, - 'properties': { - 'ref': variant.ref, - 'name': variant.name, - 'stroke': variant.colour + features.append( + { + 'type': 'Feature', + 'geometry': { + 'type': 'LineString', + 'coordinates': variant.tracks, + }, + 'properties': { + 'ref': variant.ref, + 'name': variant.name, + 'stroke': variant.colour, + }, } - }) + ) for st in variant: stops.add(st.stop) stopareas.add(st.stoparea) for stop in stops: - features.append({ - 'type': 'Feature', - 'geometry': { - 'type': 'Point', - 'coordinates': stop, - }, - 'properties': { - 'marker-size': 'small', - 'marker-symbol': 'circle' + features.append( + { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': stop, + }, + 'properties': { + 'marker-size': 'small', + 'marker-symbol': 'circle', + }, } - }) + ) for stoparea in stopareas: - features.append({ - 'type': 'Feature', - 'geometry': { - 'type': 'Point', - 'coordinates': stoparea.center, - }, - 'properties': { - 'name': stoparea.name, - 'marker-size': 'small', - 'marker-color': '#ff2600' if stoparea in transfers else '#797979' + features.append( + { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': stoparea.center, + }, + 'properties': { + 'name': stoparea.name, + 'marker-size': 'small', + 'marker-color': '#ff2600' + if stoparea in transfers + else '#797979', + }, } - }) + ) return {'type': 'FeatureCollection', 'features': features} - def _dumps_route_id(route_id): """Argument is a route_id that depends on route colour and ref. Name can be taken from route_master or can be route's own, we don't take it @@ -223,9 +248,11 @@ def read_recovery_data(path): return {} else: data = { - city_name: {_loads_route_id(route_id): route_data - for route_id, route_data in routes.items()} - for city_name, routes in data.items() + city_name: { + _loads_route_id(route_id): route_data + for route_id, route_data in routes.items() + } + for city_name, routes in data.items() } return data @@ -241,20 +268,24 @@ def write_recovery_data(path, current_data, cities): route_id = (route.colour, route.ref) itineraries = [] for variant in route: - itin = {'stations': [], - 'name': variant.name, - 'from': variant.element['tags'].get('from'), - 'to': variant.element['tags'].get('to')} + itin = { + 'stations': [], + 'name': variant.name, + 'from': variant.element['tags'].get('from'), + 'to': variant.element['tags'].get('to'), + } for stop in variant: station = stop.stoparea.station station_name = station.name if station_name == '?' and station.int_name: station_name = station.int_name - itin['stations'].append({ - 'oms_id': station.id, - 'name': station_name, - 'center': station.center - }) + itin['stations'].append( + { + 'oms_id': station.id, + 'name': station_name, + 'center': station.center, + } + ) if itin is not None: itineraries.append(itin) routes[route_id] = itineraries @@ -267,12 +298,13 @@ def write_recovery_data(path, current_data, cities): try: data = { - city_name: {_dumps_route_id(route_id): route_data - for route_id, route_data in routes.items()} + city_name: { + _dumps_route_id(route_id): route_data + for route_id, route_data in routes.items() + } for city_name, routes in data.items() } with open(path, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=2) except Exception as e: logging.warning("Cannot write recovery data to '%s': %s", path, str(e)) - diff --git a/subway_structure.py b/subway_structure.py index c5fc18e..222a9db 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -10,7 +10,7 @@ from collections import Counter, defaultdict SPREADSHEET_ID = '1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k' MAX_DISTANCE_TO_ENTRANCES = 300 # in meters MAX_DISTANCE_STOP_TO_LINE = 50 # in meters -ALLOWED_STATIONS_MISMATCH = 0.02 # part of total station count +ALLOWED_STATIONS_MISMATCH = 0.02 # part of total station count ALLOWED_TRANSFERS_MISMATCH = 0.07 # part of total interchanges count ALLOWED_ANGLE_BETWEEN_STOPS = 45 # in degrees DISALLOWED_ANGLE_BETWEEN_STOPS = 20 # in degrees @@ -24,9 +24,23 @@ MODES_OVERGROUND = set(('tram', 'bus', 'trolleybus', 'aerialway', 'ferry')) DEFAULT_MODES_RAPID = set(('subway', 'light_rail')) DEFAULT_MODES_OVERGROUND = set(('tram',)) # TODO: bus and trolleybus? ALL_MODES = MODES_RAPID | MODES_OVERGROUND -RAILWAY_TYPES = set(('rail', 'light_rail', 'subway', 'narrow_gauge', - 'funicular', 'monorail', 'tram')) -CONSTRUCTION_KEYS = ('construction', 'proposed', 'construction:railway', 'proposed:railway') +RAILWAY_TYPES = set( + ( + 'rail', + 'light_rail', + 'subway', + 'narrow_gauge', + 'funicular', + 'monorail', + 'tram', + ) +) +CONSTRUCTION_KEYS = ( + 'construction', + 'proposed', + 'construction:railway', + 'proposed:railway', +) used_entrances = set() @@ -56,27 +70,32 @@ def el_center(el): def distance(p1, p2): if p1 is None or p2 is None: - raise Exception('One of arguments to distance({}, {}) is None'.format(p1, p2)) + raise Exception( + 'One of arguments to distance({}, {}) is None'.format(p1, p2) + ) dx = math.radians(p1[0] - p2[0]) * math.cos( - 0.5 * math.radians(p1[1] + p2[1])) + 0.5 * math.radians(p1[1] + p2[1]) + ) dy = math.radians(p1[1] - p2[1]) - return 6378137 * math.sqrt(dx*dx + dy*dy) + return 6378137 * math.sqrt(dx * dx + dy * dy) def is_near(p1, p2): - return (p1[0] - 1e-8 <= p2[0] <= p1[0] + 1e-8 and - p1[1] - 1e-8 <= p2[1] <= p1[1] + 1e-8) + return ( + p1[0] - 1e-8 <= p2[0] <= p1[0] + 1e-8 + and p1[1] - 1e-8 <= p2[1] <= p1[1] + 1e-8 + ) def project_on_line(p, line): def project_on_segment(p, p1, p2): dp = (p2[0] - p1[0], p2[1] - p1[1]) - d2 = dp[0]*dp[0] + dp[1]*dp[1] + d2 = dp[0] * dp[0] + dp[1] * dp[1] if d2 < 1e-14: return None # u is the position of projection of p point on line p1p2 # regarding point p1 and (p2-p1) direction vector - u = ((p[0] - p1[0])*dp[0] + (p[1] - p1[1])*dp[1]) / d2 + u = ((p[0] - p1[0]) * dp[0] + (p[1] - p1[1]) * dp[1]) / d2 if not 0 <= u <= 1: return None return u @@ -87,7 +106,7 @@ def project_on_line(p, line): # the projected point lies on a segment between two vertices. More than # one value can occur if a route follows the same tracks more than once. 'positions_on_line': None, - 'projected_point': None # (lon, lat) + 'projected_point': None, # (lon, lat) } if len(line) < 2: @@ -106,18 +125,28 @@ def project_on_line(p, line): # Repeated occurrence of the track vertex in line, like Oslo Line 5 result['positions_on_line'].append(i) # And then calculate distances to each segment - for seg in range(len(line)-1): + for seg in range(len(line) - 1): # Check bbox for speed - if not ((min(line[seg][0], line[seg+1][0]) - MAX_DISTANCE_STOP_TO_LINE <= p[0] <= - max(line[seg][0], line[seg+1][0]) + MAX_DISTANCE_STOP_TO_LINE) and - (min(line[seg][1], line[seg+1][1]) - MAX_DISTANCE_STOP_TO_LINE <= p[1] <= - max(line[seg][1], line[seg+1][1]) + MAX_DISTANCE_STOP_TO_LINE)): + if not ( + ( + min(line[seg][0], line[seg + 1][0]) - MAX_DISTANCE_STOP_TO_LINE + <= p[0] + <= max(line[seg][0], line[seg + 1][0]) + + MAX_DISTANCE_STOP_TO_LINE + ) + and ( + min(line[seg][1], line[seg + 1][1]) - MAX_DISTANCE_STOP_TO_LINE + <= p[1] + <= max(line[seg][1], line[seg + 1][1]) + + MAX_DISTANCE_STOP_TO_LINE + ) + ): continue - u = project_on_segment(p, line[seg], line[seg+1]) + u = project_on_segment(p, line[seg], line[seg + 1]) if u: projected_point = ( - line[seg][0] + u * (line[seg+1][0] - line[seg][0]), - line[seg][1] + u * (line[seg+1][1] - line[seg][1]) + line[seg][0] + u * (line[seg + 1][0] - line[seg][0]), + line[seg][1] + u * (line[seg + 1][1] - line[seg][1]), ) d = distance(p, projected_point) if d < d_min: @@ -135,24 +164,24 @@ def project_on_line(p, line): def find_segment(p, line, start_vertex=0): """Returns index of a segment and a position inside it.""" EPS = 1e-9 - for seg in range(start_vertex, len(line)-1): + for seg in range(start_vertex, len(line) - 1): if is_near(p, line[seg]): return seg, 0 - if line[seg][0] == line[seg+1][0]: - if not (p[0]-EPS <= line[seg][0] <= p[0]+EPS): + if line[seg][0] == line[seg + 1][0]: + if not (p[0] - EPS <= line[seg][0] <= p[0] + EPS): continue px = None else: - px = (p[0] - line[seg][0]) / (line[seg+1][0] - line[seg][0]) + px = (p[0] - line[seg][0]) / (line[seg + 1][0] - line[seg][0]) if px is None or (0 <= px <= 1): - if line[seg][1] == line[seg+1][1]: - if not (p[1]-EPS <= line[seg][1] <= p[1]+EPS): + if line[seg][1] == line[seg + 1][1]: + if not (p[1] - EPS <= line[seg][1] <= p[1] + EPS): continue py = None else: - py = (p[1] - line[seg][1]) / (line[seg+1][1] - line[seg][1]) + py = (p[1] - line[seg][1]) / (line[seg + 1][1] - line[seg][1]) if py is None or (0 <= py <= 1): - if py is None or px is None or (px-EPS <= py <= px+EPS): + if py is None or px is None or (px - EPS <= py <= px + EPS): return seg, px or py return None, None @@ -176,24 +205,30 @@ def distance_on_line(p1, p2, line, start_vertex=0): # logging.warn('p2 %s is not projected, st=%s', p2, start_vertex) return None if seg1 == seg2: - return distance(line[seg1], line[seg1+1]) * abs(pos2-pos1), seg1 + return distance(line[seg1], line[seg1 + 1]) * abs(pos2 - pos1), seg1 if seg2 < seg1: # Should not happen raise Exception('Pos1 %s is after pos2 %s', seg1, seg2) d = 0 if pos1 < 1: - d += distance(line[seg1], line[seg1+1]) * (1-pos1) - for i in range(seg1+1, seg2): - d += distance(line[i], line[i+1]) + d += distance(line[seg1], line[seg1 + 1]) * (1 - pos1) + for i in range(seg1 + 1, seg2): + d += distance(line[i], line[i + 1]) if pos2 > 0: - d += distance(line[seg2], line[seg2+1]) * pos2 + d += distance(line[seg2], line[seg2 + 1]) * pos2 return d, seg2 % len(line_copy) def angle_between(p1, c, p2): - a = round(abs(math.degrees(math.atan2(p1[1]-c[1], p1[0]-c[0]) - - math.atan2(p2[1]-c[1], p2[0]-c[0])))) - return a if a <= 180 else 360-a + a = round( + abs( + math.degrees( + math.atan2(p1[1] - c[1], p1[0] - c[0]) + - math.atan2(p2[1] - c[1], p2[0] - c[0]) + ) + ) + ) + return a if a <= 180 else 360 - a def format_elid_list(ids): @@ -217,7 +252,10 @@ class Station: def is_station(el, modes): # public_transport=station is too ambiguous and unspecific to use, # so we expect for it to be backed by railway=station. - if 'tram' in modes and el.get('tags', {}).get('railway') == 'tram_stop': + if ( + 'tram' in modes + and el.get('tags', {}).get('railway') == 'tram_stop' + ): return True if el.get('tags', {}).get('railway') not in ('station', 'halt'): return False @@ -233,13 +271,17 @@ class Station: """Call this with a railway=station node.""" if not Station.is_station(el, city.modes): raise Exception( - 'Station object should be instantiated from a station node. Got: {}'.format(el)) + 'Station object should be instantiated from a station node. ' + 'Got: {}'.format(el) + ) self.id = el_id(el) self.element = el self.modes = Station.get_modes(el) self.name = el['tags'].get('name', '?') - self.int_name = el['tags'].get('int_name', el['tags'].get('name:en', None)) + self.int_name = el['tags'].get( + 'int_name', el['tags'].get('name:en', None) + ) try: self.colour = normalize_colour(el['tags'].get('colour', None)) except ValueError as e: @@ -251,7 +293,8 @@ class Station: def __repr__(self): return 'Station(id={}, modes={}, name={}, center={})'.format( - self.id, ','.join(self.modes), self.name, self.center) + self.id, ','.join(self.modes), self.name, self.center + ) class StopArea: @@ -287,13 +330,14 @@ class StopArea: self.element = stop_area or station.element self.id = el_id(self.element) self.station = station - self.stops = set() # set of el_ids of stop_positions + self.stops = set() # set of el_ids of stop_positions self.platforms = set() # set of el_ids of platforms - self.exits = set() # el_id of subway_entrance for leaving the platform - self.entrances = set() # el_id of subway_entrance for entering the platform - self.center = None # lon, lat of the station centre point - self.centers = {} # el_id -> (lon, lat) for all elements - self.transfer = None # el_id of a transfer relation + self.exits = set() # el_id of subway_entrance for leaving the platform + self.entrances = set() # el_id of subway_entrance for entering + # the platform + self.center = None # lon, lat of the station centre point + self.centers = {} # el_id -> (lon, lat) for all elements + self.transfer = None # el_id of a transfer relation self.modes = station.modes self.name = station.name @@ -303,10 +347,13 @@ class StopArea: if stop_area: self.name = stop_area['tags'].get('name', self.name) self.int_name = stop_area['tags'].get( - 'int_name', stop_area['tags'].get('name:en', self.int_name)) + 'int_name', stop_area['tags'].get('name:en', self.int_name) + ) try: - self.colour = normalize_colour( - stop_area['tags'].get('colour')) or self.colour + self.colour = ( + normalize_colour(stop_area['tags'].get('colour')) + or self.colour + ) except ValueError as e: city.warn(str(e), stop_area) @@ -318,7 +365,9 @@ class StopArea: if m_el and 'tags' in m_el: if Station.is_station(m_el, city.modes): if k != station.id: - city.error('Stop area has multiple stations', stop_area) + city.error( + 'Stop area has multiple stations', stop_area + ) elif StopArea.is_stop(m_el): self.stops.add(k) elif StopArea.is_platform(m_el): @@ -326,13 +375,21 @@ class StopArea: elif m_el['tags'].get('railway') == 'subway_entrance': if m_el['type'] != 'node': city.warn('Subway entrance is not a node', m_el) - if m_el['tags'].get('entrance') != 'exit' and m['role'] != 'exit_only': + if ( + m_el['tags'].get('entrance') != 'exit' + and m['role'] != 'exit_only' + ): self.entrances.add(k) - if m_el['tags'].get('entrance') != 'entrance' and m['role'] != 'entry_only': + if ( + m_el['tags'].get('entrance') != 'entrance' + and m['role'] != 'entry_only' + ): self.exits.add(k) elif StopArea.is_track(m_el): if not warned_about_tracks: - city.error('Tracks in a stop_area relation', stop_area) + city.error( + 'Tracks in a stop_area relation', stop_area + ) warned_about_tracks = True else: # Otherwise add nearby entrances @@ -342,9 +399,15 @@ class StopArea: c_id = el_id(c_el) if c_id not in city.stop_areas: c_center = el_center(c_el) - if c_center and distance(center, c_center) <= MAX_DISTANCE_TO_ENTRANCES: + if ( + c_center + and distance(center, c_center) + <= MAX_DISTANCE_TO_ENTRANCES + ): if c_el['type'] != 'node': - city.warn('Subway entrance is not a node', c_el) + city.warn( + 'Subway entrance is not a node', c_el + ) etag = c_el['tags'].get('entrance') if etag != 'exit': self.entrances.add(c_id) @@ -352,7 +415,10 @@ class StopArea: self.exits.add(c_id) if self.exits and not self.entrances: - city.error('Only exits for a station, no entrances', stop_area or station.element) + city.error( + 'Only exits for a station, no entrances', + stop_area or station.element, + ) if self.entrances and not self.exits: city.error('No exits for a station', stop_area or station.element) @@ -384,7 +450,8 @@ class StopArea: def __repr__(self): return 'StopArea(id={}, name={}, station={}, transfer={}, center={})'.format( - self.id, self.name, self.station, self.transfer, self.center) + self.id, self.name, self.station, self.transfer, self.center + ) class RouteStop: @@ -468,7 +535,12 @@ class RouteStop: self.seen_platform_exit = True else: if role != 'platform' and 'stop' not in role: - city.warn("Platform with invalid role '{}' in a route".format(role), el) + city.warn( + "Platform with invalid role '{}' in a route".format( + role + ), + el, + ) multiple_check = self.seen_platform self.seen_platform_entry = True self.seen_platform_exit = True @@ -479,18 +551,31 @@ class RouteStop: city.error_if( actual_role == 'stop', 'Multiple {}s for a station "{}" ({}) in a route relation'.format( - actual_role, el['tags'].get('name', ''), el_id(el)), relation) + actual_role, el['tags'].get('name', ''), el_id(el) + ), + relation, + ) def __repr__(self): - return 'RouteStop(stop={}, pl_entry={}, pl_exit={}, stoparea={})'.format( - self.stop, self.platform_entry, self.platform_exit, self.stoparea) + return ( + 'RouteStop(stop={}, pl_entry={}, pl_exit={}, stoparea={})'.format( + self.stop, + self.platform_entry, + self.platform_exit, + self.stoparea, + ) + ) class Route: """The longest route for a city with a unique ref.""" + @staticmethod def is_route(el, modes): - if el['type'] != 'relation' or el.get('tags', {}).get('type') != 'route': + if ( + el['type'] != 'relation' + or el.get('tags', {}).get('type') != 'route' + ): return False if 'members' not in el: return False @@ -519,7 +604,7 @@ class Route: break else: for kk in tags: - if kk.startswith(k+':'): + if kk.startswith(k + ':'): v = tags[kk] break if not v: @@ -554,7 +639,10 @@ class Route: track.extend(new_segment[1:]) elif new_segment[-1] == track[-1]: track.extend(reversed(new_segment[:-1])) - elif is_first and track[0] in (new_segment[0], new_segment[-1]): + elif is_first and track[0] in ( + new_segment[0], + new_segment[-1], + ): # We can reverse the track and try again track.reverse() if new_segment[0] == track[-1]: @@ -564,8 +652,12 @@ class Route: else: # Store the track if it is long and clean it if not warned_about_holes: - self.city.warn('Hole in route rails near node {}'.format( - track[-1]), relation) + self.city.warn( + 'Hole in route rails near node {}'.format( + track[-1] + ), + relation, + ) warned_about_holes = True if len(track) > len(last_track): last_track = track @@ -574,8 +666,11 @@ class Route: if len(track) > len(last_track): last_track = track # Remove duplicate points - last_track = [last_track[i] for i in range(0, len(last_track)) - if i == 0 or last_track[i-1] != last_track[i]] + last_track = [ + last_track[i] + for i in range(0, len(last_track)) + if i == 0 or last_track[i - 1] != last_track[i] + ] return last_track, line_nodes def project_stops_on_line(self): @@ -583,11 +678,12 @@ class Route: def is_stop_near_tracks(stop_index): return ( - projected[stop_index]['projected_point'] is not None and - distance( - self.stops[stop_index].stop, - projected[stop_index]['projected_point'] - ) <= MAX_DISTANCE_STOP_TO_LINE + projected[stop_index]['projected_point'] is not None + and distance( + self.stops[stop_index].stop, + projected[stop_index]['projected_point'], + ) + <= MAX_DISTANCE_STOP_TO_LINE ) start = 0 @@ -605,19 +701,29 @@ class Route: elif i > end: tracks_end.append(route_stop.stop) elif projected[i]['projected_point'] is None: - self.city.error('Stop "{}" {} is nowhere near the tracks'.format( - route_stop.stoparea.name, route_stop.stop), self.element) + self.city.error( + 'Stop "{}" {} is nowhere near the tracks'.format( + route_stop.stoparea.name, route_stop.stop + ), + self.element, + ) else: projected_point = projected[i]['projected_point'] # We've got two separate stations with a good stretch of # railway tracks between them. Put these on tracks. d = round(distance(route_stop.stop, projected_point)) if d > MAX_DISTANCE_STOP_TO_LINE: - self.city.warn('Stop "{}" {} is {} meters from the tracks'.format( - route_stop.stoparea.name, route_stop.stop, d), self.element) + self.city.warn( + 'Stop "{}" {} is {} meters from the tracks'.format( + route_stop.stoparea.name, route_stop.stop, d + ), + self.element, + ) else: route_stop.stop = projected_point - route_stop.positions_on_rails = projected[i]['positions_on_line'] + route_stop.positions_on_rails = projected[i][ + 'positions_on_line' + ] stops_on_longest_line.append(route_stop) if start >= len(self.stops): self.tracks = tracks_start @@ -630,9 +736,11 @@ class Route: vertex = 0 for i, stop in enumerate(self.stops): if i > 0: - direct = distance(stop.stop, self.stops[i-1].stop) - d_line = distance_on_line(self.stops[i-1].stop, stop.stop, self.tracks, vertex) - if d_line and direct-10 <= d_line[0] <= direct*2: + direct = distance(stop.stop, self.stops[i - 1].stop) + d_line = distance_on_line( + self.stops[i - 1].stop, stop.stop, self.tracks, vertex + ) + if d_line and direct - 10 <= d_line[0] <= direct * 2: vertex = d_line[1] dist += round(d_line[0]) else: @@ -641,46 +749,71 @@ class Route: def __init__(self, relation, city, master=None): if not Route.is_route(relation, city.modes): - raise Exception('The relation does not seem a route: {}'.format(relation)) + raise Exception( + 'The relation does not seem a route: {}'.format(relation) + ) master_tags = {} if not master else master['tags'] self.city = city self.element = relation self.id = el_id(relation) if 'ref' not in relation['tags'] and 'ref' not in master_tags: city.warn('Missing ref on a route', relation) - self.ref = relation['tags'].get('ref', master_tags.get( - 'ref', relation['tags'].get('name', None))) + self.ref = relation['tags'].get( + 'ref', master_tags.get('ref', relation['tags'].get('name', None)) + ) self.name = relation['tags'].get('name', None) self.mode = relation['tags']['route'] - if 'colour' not in relation['tags'] and 'colour' not in master_tags and self.mode != 'tram': + if ( + 'colour' not in relation['tags'] + and 'colour' not in master_tags + and self.mode != 'tram' + ): city.warn('Missing colour on a route', relation) try: - self.colour = normalize_colour(relation['tags'].get( - 'colour', master_tags.get('colour', None))) + self.colour = normalize_colour( + relation['tags'].get('colour', master_tags.get('colour', None)) + ) except ValueError as e: self.colour = None city.warn(str(e), relation) try: - self.infill = normalize_colour(relation['tags'].get( - 'colour:infill', master_tags.get('colour:infill', None))) + self.infill = normalize_colour( + relation['tags'].get( + 'colour:infill', master_tags.get('colour:infill', None) + ) + ) except ValueError as e: self.infill = None city.warn(str(e), relation) self.network = Route.get_network(relation) - self.interval = Route.get_interval(relation['tags']) or Route.get_interval(master_tags) + self.interval = Route.get_interval( + relation['tags'] + ) or Route.get_interval(master_tags) if relation['tags'].get('public_transport:version') == '1': - city.error('Public transport version is 1, which means the route ' - 'is an unsorted pile of objects', relation) + city.error( + 'Public transport version is 1, which means the route ' + 'is an unsorted pile of objects', + relation, + ) self.is_circular = False # self.tracks would be a list of (lon, lat) for the longest stretch. Can be empty tracks, line_nodes = self.build_longest_line(relation) self.tracks = [el_center(city.elements.get(k)) for k in tracks] - if None in self.tracks: # usually, extending BBOX for the city is needed + if ( + None in self.tracks + ): # usually, extending BBOX for the city is needed self.tracks = [] for n in filter(lambda x: x not in city.elements, tracks): - city.error('The dataset is missing the railway tracks node {}'.format(n), relation) + city.error( + 'The dataset is missing the railway tracks node {}'.format( + n + ), + relation, + ) break - check_stop_positions = len(line_nodes) > 50 # arbitrary number, of course + check_stop_positions = ( + len(line_nodes) > 50 + ) # arbitrary number, of course self.stops = [] # List of RouteStop stations = set() # temporary for recording stations seen_stops = False @@ -694,13 +827,23 @@ class Route: st_list = city.stations[k] st = st_list[0] if len(st_list) > 1: - city.error('Ambiguous station {} in route. Please use stop_position or split ' - 'interchange stations'.format(st.name), relation) + city.error( + 'Ambiguous station {} in route. Please use stop_position or split ' + 'interchange stations'.format(st.name), + relation, + ) el = city.elements[k] - actual_role = RouteStop.get_actual_role(el, m['role'], city.modes) + actual_role = RouteStop.get_actual_role( + el, m['role'], city.modes + ) if actual_role: if m['role'] and actual_role not in m['role']: - city.warn("Wrong role '{}' for {} {}".format(m['role'], actual_role, k), relation) + city.warn( + "Wrong role '{}' for {} {}".format( + m['role'], actual_role, k + ), + relation, + ) if repeat_pos is None: if not self.stops or st not in stations: stop = RouteStop(st) @@ -710,9 +853,17 @@ class Route: stop = self.stops[-1] else: # We've got a repeat - if ((seen_stops and seen_platforms) or - (actual_role == 'stop' and not seen_platforms) or - (actual_role == 'platform' and not seen_stops)): + if ( + (seen_stops and seen_platforms) + or ( + actual_role == 'stop' + and not seen_platforms + ) + or ( + actual_role == 'platform' + and not seen_stops + ) + ): # Circular route! stop = RouteStop(st) self.stops.append(stop) @@ -724,17 +875,28 @@ class Route: continue # Check that the type matches if (actual_role == 'stop' and seen_stops) or ( - actual_role == 'platform' and seen_platforms): - city.error('Found an out-of-place {}: "{}" ({})'.format( - actual_role, el['tags'].get('name', ''), k), relation) + actual_role == 'platform' and seen_platforms + ): + city.error( + 'Found an out-of-place {}: "{}" ({})'.format( + actual_role, el['tags'].get('name', ''), k + ), + relation, + ) continue # Find the matching stop starting with index repeat_pos - while (repeat_pos < len(self.stops) and - self.stops[repeat_pos].stoparea.id != st.id): + while ( + repeat_pos < len(self.stops) + and self.stops[repeat_pos].stoparea.id != st.id + ): repeat_pos += 1 if repeat_pos >= len(self.stops): - city.error('Incorrect order of {}s at {}'.format(actual_role, k), - relation) + city.error( + 'Incorrect order of {}s at {}'.format( + actual_role, k + ), + relation, + ) continue stop = self.stops[repeat_pos] @@ -745,8 +907,12 @@ class Route: if check_stop_positions and StopArea.is_stop(el): if k not in line_nodes: - city.warn('Stop position "{}" ({}) is not on tracks'.format( - el['tags'].get('name', ''), k), relation) + city.warn( + 'Stop position "{}" ({}) is not on tracks'.format( + el['tags'].get('name', ''), k + ), + relation, + ) continue if k not in city.elements: @@ -765,9 +931,13 @@ class Route: is_under_construction = False for ck in CONSTRUCTION_KEYS: if ck in el['tags']: - city.error('An under construction {} {} in route. Consider ' - 'setting \'inactive\' role or removing construction attributes' - .format(m['role'] or 'feature', k), relation) + city.error( + 'An under construction {} {} in route. Consider ' + 'setting \'inactive\' role or removing construction attributes'.format( + m['role'] or 'feature', k + ), + relation, + ) is_under_construction = True break if is_under_construction: @@ -778,20 +948,36 @@ class Route: # 'stop area has multiple stations' error. No other error message is needed. pass elif el['tags'].get('railway') in ('station', 'halt'): - city.error('Missing station={} on a {}'.format(self.mode, m['role']), el) + city.error( + 'Missing station={} on a {}'.format(self.mode, m['role']), + el, + ) else: - actual_role = RouteStop.get_actual_role(el, m['role'], city.modes) + actual_role = RouteStop.get_actual_role( + el, m['role'], city.modes + ) if actual_role: - city.error('{} {} {} is not connected to a station in route'.format( - actual_role, m['type'], m['ref']), relation) + city.error( + '{} {} {} is not connected to a station in route'.format( + actual_role, m['type'], m['ref'] + ), + relation, + ) elif not StopArea.is_track(el): - city.error('Unknown member type for {} {} in route'.format(m['type'], m['ref']), relation) + city.error( + 'Unknown member type for {} {} in route'.format( + m['type'], m['ref'] + ), + relation, + ) if not self.stops: city.error('Route has no stops', relation) elif len(self.stops) == 1: city.error('Route has only one stop', relation) else: - self.is_circular = self.stops[0].stoparea == self.stops[-1].stoparea + self.is_circular = ( + self.stops[0].stoparea == self.stops[-1].stoparea + ) stops_on_longest_line = self.project_stops_on_line() self.check_and_recover_stops_order(stops_on_longest_line) self.calculate_distances() @@ -800,12 +986,15 @@ class Route: disorder_warnings = [] disorder_errors = [] for si in range(len(self.stops) - 2): - angle = angle_between(self.stops[si].stop, - self.stops[si + 1].stop, - self.stops[si + 2].stop) + angle = angle_between( + self.stops[si].stop, + self.stops[si + 1].stop, + self.stops[si + 2].stop, + ) if angle < ALLOWED_ANGLE_BETWEEN_STOPS: msg = 'Angle between stops around "{}" is too narrow, {} degrees'.format( - self.stops[si + 1].stoparea.name, angle) + self.stops[si + 1].stoparea.name, angle + ) if angle < DISALLOWED_ANGLE_BETWEEN_STOPS: disorder_errors.append(msg) else: @@ -819,6 +1008,7 @@ class Route: longest contiguous sequence of tracks in a route. :return: error message on the first order violation or None. """ + def make_assertion_error_msg(route_stop, error_type): return ( "stop_area {} '{}' has {} 'positions_on_rails' " @@ -826,23 +1016,28 @@ class Route: route_stop.stoparea.id, route_stop.stoparea.name, "no" if error_type == 1 else "empty", - self.id + self.id, ) ) allowed_order_violations = 1 if self.is_circular else 0 max_position_on_rails = -1 for route_stop in stop_sequence: - assert hasattr(route_stop, 'positions_on_rails'), \ - make_assertion_error_msg(route_stop, error_type=1) + assert hasattr( + route_stop, 'positions_on_rails' + ), make_assertion_error_msg(route_stop, error_type=1) positions_on_rails = route_stop.positions_on_rails - assert positions_on_rails, \ - make_assertion_error_msg(route_stop, error_type=2) + assert positions_on_rails, make_assertion_error_msg( + route_stop, error_type=2 + ) suitable_occurrence = 0 - while (suitable_occurrence < len(positions_on_rails) and - positions_on_rails[suitable_occurrence] < max_position_on_rails): + while ( + suitable_occurrence < len(positions_on_rails) + and positions_on_rails[suitable_occurrence] + < max_position_on_rails + ): suitable_occurrence += 1 if suitable_occurrence == len(positions_on_rails): if allowed_order_violations > 0: @@ -850,13 +1045,12 @@ class Route: allowed_order_violations -= 1 else: return 'Stops on tracks are unordered near "{}" {}'.format( - route_stop.stoparea.name, - route_stop.stop + route_stop.stoparea.name, route_stop.stop ) max_position_on_rails = positions_on_rails[suitable_occurrence] def check_stops_order_on_tracks(self, stop_sequence): - """ Checks stops order on tracks, trying direct and reversed + """Checks stops order on tracks, trying direct and reversed order of stops in the stop_sequence. :param stop_sequence: list of RouteStop that belong to the longest contiguous sequence of tracks in a route. @@ -864,15 +1058,25 @@ class Route: """ error_message = self.check_stops_order_on_tracks_direct(stop_sequence) if error_message: - error_message_reversed = self.check_stops_order_on_tracks_direct(reversed(stop_sequence)) + error_message_reversed = self.check_stops_order_on_tracks_direct( + reversed(stop_sequence) + ) if error_message_reversed is None: error_message = None - self.city.warn('Tracks seem to go in the opposite direction to stops', self.element) + self.city.warn( + 'Tracks seem to go in the opposite direction to stops', + self.element, + ) return error_message def check_stops_order(self, stops_on_longest_line): - angle_disorder_warnings, angle_disorder_errors = self.check_stops_order_by_angle() - disorder_on_tracks_error = self.check_stops_order_on_tracks(stops_on_longest_line) + ( + angle_disorder_warnings, + angle_disorder_errors, + ) = self.check_stops_order_by_angle() + disorder_on_tracks_error = self.check_stops_order_on_tracks( + stops_on_longest_line + ) disorder_warnings = angle_disorder_warnings disorder_errors = angle_disorder_errors if disorder_on_tracks_error: @@ -880,7 +1084,9 @@ class Route: return disorder_warnings, disorder_errors def check_and_recover_stops_order(self, stops_on_longest_line): - disorder_warnings, disorder_errors = self.check_stops_order(stops_on_longest_line) + disorder_warnings, disorder_errors = self.check_stops_order( + stops_on_longest_line + ) if disorder_warnings or disorder_errors: resort_success = False if self.city.recovery_data: @@ -889,7 +1095,9 @@ class Route: for msg in disorder_warnings: self.city.warn(msg, self.element) for msg in disorder_errors: - self.city.warn("Fixed with recovery data: " + msg, self.element) + self.city.warn( + "Fixed with recovery data: " + msg, self.element + ) if not resort_success: for msg in disorder_warnings: @@ -900,7 +1108,7 @@ class Route: def try_resort_stops(self): """Precondition: self.city.recovery_data is not None. Return success of station order recovering.""" - self_stops = {} # station name => RouteStop + self_stops = {} # station name => RouteStop for stop in self.stops: station = stop.stoparea.station stop_name = station.name @@ -919,16 +1127,23 @@ class Route: stop_names = list(self_stops.keys()) suitable_itineraries = [] for itinerary in self.city.recovery_data[route_id]: - itinerary_stop_names = [stop['name'] for stop in itinerary['stations']] - if not (len(stop_names) == len(itinerary_stop_names) and - sorted(stop_names) == sorted(itinerary_stop_names)): + itinerary_stop_names = [ + stop['name'] for stop in itinerary['stations'] + ] + if not ( + len(stop_names) == len(itinerary_stop_names) + and sorted(stop_names) == sorted(itinerary_stop_names) + ): continue big_station_displacement = False for it_stop in itinerary['stations']: name = it_stop['name'] it_stop_center = it_stop['center'] self_stop_center = self_stops[name].stoparea.station.center - if distance(it_stop_center, self_stop_center) > DISPLACEMENT_TOLERANCE: + if ( + distance(it_stop_center, self_stop_center) + > DISPLACEMENT_TOLERANCE + ): big_station_displacement = True break if not big_station_displacement: @@ -944,14 +1159,19 @@ class Route: if not from_tag and not to_tag: return False matching_itineraries = [ - itin for itin in suitable_itineraries - if from_tag and itin['from'] == from_tag or - to_tag and itin['to'] == to_tag + itin + for itin in suitable_itineraries + if from_tag + and itin['from'] == from_tag + or to_tag + and itin['to'] == to_tag ] if len(matching_itineraries) != 1: return False matching_itinerary = matching_itineraries[0] - self.stops = [self_stops[stop['name']] for stop in matching_itinerary['stations']] + self.stops = [ + self_stops[stop['name']] for stop in matching_itinerary['stations'] + ] return True def __len__(self): @@ -964,11 +1184,22 @@ class Route: return iter(self.stops) def __repr__(self): - return ('Route(id={}, mode={}, ref={}, name={}, network={}, interval={}, ' - 'circular={}, num_stops={}, line_length={} m, from={}, to={}').format( - self.id, self.mode, self.ref, self.name, self.network, self.interval, - self.is_circular, len(self.stops), self.stops[-1].distance, - self.stops[0], self.stops[-1]) + return ( + 'Route(id={}, mode={}, ref={}, name={}, network={}, interval={}, ' + 'circular={}, num_stops={}, line_length={} m, from={}, to={}' + ).format( + self.id, + self.mode, + self.ref, + self.name, + self.network, + self.interval, + self.is_circular, + len(self.stops), + self.stops[-1].distance, + self.stops[0], + self.stops[-1], + ) class RouteMaster: @@ -979,17 +1210,25 @@ class RouteMaster: self.has_master = master is not None self.interval_from_master = False if master: - self.ref = master['tags'].get('ref', master['tags'].get('name', None)) + self.ref = master['tags'].get( + 'ref', master['tags'].get('name', None) + ) try: - self.colour = normalize_colour(master['tags'].get('colour', None)) + self.colour = normalize_colour( + master['tags'].get('colour', None) + ) except ValueError: self.colour = None try: - self.infill = normalize_colour(master['tags'].get('colour:infill', None)) + self.infill = normalize_colour( + master['tags'].get('colour:infill', None) + ) except ValueError: self.colour = None self.network = Route.get_network(master) - self.mode = master['tags'].get('route_master', None) # This tag is required, but okay + self.mode = master['tags'].get( + 'route_master', None + ) # This tag is required, but okay self.name = master['tags'].get('name', None) self.interval = Route.get_interval(master['tags']) self.interval_from_master = self.interval is not None @@ -1006,26 +1245,42 @@ class RouteMaster: if not self.network: self.network = route.network elif route.network and route.network != self.network: - city.error('Route has different network ("{}") from master "{}"'.format( - route.network, self.network), route.element) + city.error( + 'Route has different network ("{}") from master "{}"'.format( + route.network, self.network + ), + route.element, + ) if not self.colour: self.colour = route.colour elif route.colour and route.colour != self.colour: - city.warn('Route "{}" has different colour from master "{}"'.format( - route.colour, self.colour), route.element) + city.warn( + 'Route "{}" has different colour from master "{}"'.format( + route.colour, self.colour + ), + route.element, + ) if not self.infill: self.infill = route.infill elif route.infill and route.infill != self.infill: - city.warn('Route "{}" has different infill colour from master "{}"'.format( - route.infill, self.infill), route.element) + city.warn( + 'Route "{}" has different infill colour from master "{}"'.format( + route.infill, self.infill + ), + route.element, + ) if not self.ref: self.ref = route.ref elif route.ref != self.ref: - city.warn('Route "{}" has different ref from master "{}"'.format( - route.ref, self.ref), route.element) + city.warn( + 'Route "{}" has different ref from master "{}"'.format( + route.ref, self.ref + ), + route.element, + ) if not self.name: self.name = route.name @@ -1033,8 +1288,12 @@ class RouteMaster: if not self.mode: self.mode = route.mode elif route.mode != self.mode: - city.error('Incompatible PT mode: master has {} and route has {}'.format( - self.mode, route.mode), route.element) + city.error( + 'Incompatible PT mode: master has {} and route has {}'.format( + self.mode, route.mode + ), + route.element, + ) return if not self.interval_from_master and route.interval: @@ -1071,7 +1330,13 @@ class RouteMaster: def __repr__(self): return 'RouteMaster(id={}, mode={}, ref={}, name={}, network={}, num_variants={}'.format( - self.id, self.mode, self.ref, self.name, self.network, len(self.routes)) + self.id, + self.mode, + self.ref, + self.name, + self.network, + len(self.routes), + ) class City: @@ -1101,7 +1366,9 @@ class City: if not networks or len(networks[-1]) == 0: self.networks = [] else: - self.networks = set(filter(None, [x.strip() for x in networks[-1].split(';')])) + self.networks = set( + filter(None, [x.strip() for x in networks[-1].split(';')]) + ) if not networks or len(networks) < 2 or len(networks[0]) == 0: if self.overground: self.modes = DEFAULT_MODES_OVERGROUND @@ -1117,11 +1384,13 @@ class City: else: self.bbox = None - self.elements = {} # Dict el_id → el - self.stations = defaultdict(list) # Dict el_id → list of StopAreas - self.routes = {} # Dict route_ref → route - self.masters = {} # Dict el_id of route → route_master - self.stop_areas = defaultdict(list) # El_id → list of el_id of stop_area + self.elements = {} # Dict el_id → el + self.stations = defaultdict(list) # Dict el_id → list of StopAreas + self.routes = {} # Dict route_ref → route + self.masters = {} # Dict el_id of route → route_master + self.stop_areas = defaultdict( + list + ) # El_id → list of el_id of stop_area self.transfers = [] # List of lists of stop areas self.station_ids = set() # Set of stations' uid self.stops_and_platforms = set() # Set of stops and platforms el_id @@ -1131,8 +1400,10 @@ class City: if el: tags = el.get('tags', {}) message += ' ({} {}, "{}")'.format( - el['type'], el.get('id', el.get('ref')), - tags.get('name', tags.get('ref', ''))) + el['type'], + el.get('id', el.get('ref')), + tags.get('name', tags.get('ref', '')), + ) return message def warn(self, message, el=None): @@ -1152,8 +1423,10 @@ class City: def contains(self, el): center = el_center(el) if center: - return (self.bbox[0] <= center[1] <= self.bbox[2] and - self.bbox[1] <= center[0] <= self.bbox[3]) + return ( + self.bbox[0] <= center[1] <= self.bbox[2] + and self.bbox[1] <= center[0] <= self.bbox[3] + ) return False def add(self, el): @@ -1188,17 +1461,25 @@ class City: # the sag does - near the city bbox boundary continue if 'tags' not in el: - self.error('An untagged object {} in a stop_area_group'.format(k), sag) + self.error( + 'An untagged object {} in a stop_area_group'.format(k), sag + ) continue - if (el['type'] != 'relation' or - el['tags'].get('type') != 'public_transport' or - el['tags'].get('public_transport') != 'stop_area'): + if ( + el['type'] != 'relation' + or el['tags'].get('type') != 'public_transport' + or el['tags'].get('public_transport') != 'stop_area' + ): continue if k in self.stations: stoparea = self.stations[k][0] transfer.add(stoparea) if stoparea.transfer: - self.error('Stop area {} belongs to multiple interchanges'.format(k)) + self.error( + 'Stop area {} belongs to multiple interchanges'.format( + k + ) + ) stoparea.transfer = el_id(sag) if len(transfer) > 1: self.transfers.append(transfer) @@ -1208,10 +1489,17 @@ class City: processed_stop_areas = set() for el in self.elements.values(): if Station.is_station(el, self.modes): - # See PR https://github.com/mapsme/subways/pull/98 - if el['type'] == 'relation' and el['tags'].get('type') != 'multipolygon': - self.error("A railway station cannot be a relation of type '{}'".format( - el['tags'].get('type')), el) + # See PR https://github.com/mapsme/subways/pull/98 + if ( + el['type'] == 'relation' + and el['tags'].get('type') != 'multipolygon' + ): + self.error( + "A railway station cannot be a relation of type '{}'".format( + el['tags'].get('type') + ), + el, + ) continue st = Station(el, self) self.station_ids.add(st.id) @@ -1231,8 +1519,10 @@ class City: # Check that stops and platforms belong to single stop_area for sp in station.stops | station.platforms: if sp in self.stops_and_platforms: - self.warn('A stop or a platform {} belongs to multiple ' - 'stations, might be correct'.format(sp)) + self.warn( + 'A stop or a platform {} belongs to multiple ' + 'stations, might be correct'.format(sp) + ) else: self.stops_and_platforms.add(sp) @@ -1249,7 +1539,10 @@ class City: master_network = Route.get_network(master) else: master_network = None - if network not in self.networks and master_network not in self.networks: + if ( + network not in self.networks + and master_network not in self.networks + ): continue route = Route(el, self, master) @@ -1263,8 +1556,11 @@ class City: del self.routes[k] # And while we're iterating over relations, find interchanges - if (el['type'] == 'relation' and - el.get('tags', {}).get('public_transport', None) == 'stop_area_group'): + if ( + el['type'] == 'relation' + and el.get('tags', {}).get('public_transport', None) + == 'stop_area_group' + ): self.make_transfer(el) # Filter transfers, leaving only stations that belong to routes @@ -1293,30 +1589,36 @@ class City: 'stations_found': getattr(self, 'found_stations', 0), 'transfers_found': getattr(self, 'found_interchanges', 0), 'unused_entrances': getattr(self, 'unused_entrances', 0), - 'networks': getattr(self, 'found_networks', 0) + 'networks': getattr(self, 'found_networks', 0), } if not self.overground: - result.update({ - 'subwayl_expected': self.num_lines, - 'lightrl_expected': self.num_light_lines, - 'subwayl_found': getattr(self, 'found_lines', 0), - 'lightrl_found': getattr(self, 'found_light_lines', 0), - 'stations_expected': self.num_stations, - 'transfers_expected': self.num_interchanges, - }) + result.update( + { + 'subwayl_expected': self.num_lines, + 'lightrl_expected': self.num_light_lines, + 'subwayl_found': getattr(self, 'found_lines', 0), + 'lightrl_found': getattr(self, 'found_light_lines', 0), + 'stations_expected': self.num_stations, + 'transfers_expected': self.num_interchanges, + } + ) else: - result.update({ - 'stations_expected': 0, - 'transfers_expected': 0, - 'busl_expected': self.num_bus_lines, - 'trolleybusl_expected': self.num_trolleybus_lines, - 'traml_expected': self.num_tram_lines, - 'otherl_expected': self.num_other_lines, - 'busl_found': getattr(self, 'found_bus_lines', 0), - 'trolleybusl_found': getattr(self, 'found_trolleybus_lines', 0), - 'traml_found': getattr(self, 'found_tram_lines', 0), - 'otherl_found': getattr(self, 'found_other_lines', 0) - }) + result.update( + { + 'stations_expected': 0, + 'transfers_expected': 0, + 'busl_expected': self.num_bus_lines, + 'trolleybusl_expected': self.num_trolleybus_lines, + 'traml_expected': self.num_tram_lines, + 'otherl_expected': self.num_other_lines, + 'busl_found': getattr(self, 'found_bus_lines', 0), + 'trolleybusl_found': getattr( + self, 'found_trolleybus_lines', 0 + ), + 'traml_found': getattr(self, 'found_tram_lines', 0), + 'otherl_found': getattr(self, 'found_other_lines', 0), + } + ) result['warnings'] = self.warnings result['errors'] = self.errors return result @@ -1325,15 +1627,21 @@ class City: global used_entrances stop_areas = set() for el in self.elements.values(): - if (el['type'] == 'relation' and 'tags' in el and - el['tags'].get('public_transport') == 'stop_area' and - 'members' in el): + if ( + el['type'] == 'relation' + and 'tags' in el + and el['tags'].get('public_transport') == 'stop_area' + and 'members' in el + ): stop_areas.update([el_id(m) for m in el['members']]) unused = [] not_in_sa = [] for el in self.elements.values(): - if (el['type'] == 'node' and 'tags' in el and - el['tags'].get('railway') == 'subway_entrance'): + if ( + el['type'] == 'node' + and 'tags' in el + and el['tags'].get('railway') == 'subway_entrance' + ): i = el_id(el) if i in self.stations: used_entrances.add(i) @@ -1344,11 +1652,17 @@ class City: self.unused_entrances = len(unused) self.entrances_not_in_stop_areas = len(not_in_sa) if unused: - self.warn('Found {} entrances not used in routes or stop_areas: {}'.format( - len(unused), format_elid_list(unused))) + self.warn( + 'Found {} entrances not used in routes or stop_areas: {}'.format( + len(unused), format_elid_list(unused) + ) + ) if not_in_sa: - self.warn('{} subway entrances are not in stop_area relations: {}'.format( - len(not_in_sa), format_elid_list(not_in_sa))) + self.warn( + '{} subway entrances are not in stop_area relations: {}'.format( + len(not_in_sa), format_elid_list(not_in_sa) + ) + ) def check_return_routes(self, rmaster): variants = {} @@ -1362,8 +1676,10 @@ class City: if variant[0].stoparea.transfer == variant[-1].stoparea.transfer: t = (variant[0].stoparea.id, variant[-1].stoparea.id) else: - t = (variant[0].stoparea.transfer or variant[0].stoparea.id, - variant[-1].stoparea.transfer or variant[-1].stoparea.id) + t = ( + variant[0].stoparea.transfer or variant[0].stoparea.id, + variant[-1].stoparea.transfer or variant[-1].stoparea.id, + ) if t in variants: continue variants[t] = variant.element @@ -1373,36 +1689,64 @@ class City: have_return.add(tr) if len(variants) == 0: - self.error('An empty route master {}. Please set construction:route ' - 'if it is under construction'.format(rmaster.id)) + self.error( + 'An empty route master {}. Please set construction:route ' + 'if it is under construction'.format(rmaster.id) + ) elif len(variants) == 1: - self.error_if(not rmaster.best.is_circular, 'Only one route in route_master. ' - 'Please check if it needs a return route', rmaster.best.element) + self.error_if( + not rmaster.best.is_circular, + 'Only one route in route_master. ' + 'Please check if it needs a return route', + rmaster.best.element, + ) else: for t, rel in variants.items(): if t not in have_return: self.warn('Route does not have a return direction', rel) def validate_lines(self): - self.found_light_lines = len([x for x in self.routes.values() if x.mode != 'subway']) + self.found_light_lines = len( + [x for x in self.routes.values() if x.mode != 'subway'] + ) self.found_lines = len(self.routes) - self.found_light_lines if self.found_lines != self.num_lines: - self.error('Found {} subway lines, expected {}'.format( - self.found_lines, self.num_lines)) + self.error( + 'Found {} subway lines, expected {}'.format( + self.found_lines, self.num_lines + ) + ) if self.found_light_lines != self.num_light_lines: - self.error('Found {} light rail lines, expected {}'.format( - self.found_light_lines, self.num_light_lines)) + self.error( + 'Found {} light rail lines, expected {}'.format( + self.found_light_lines, self.num_light_lines + ) + ) def validate_overground_lines(self): - self.found_tram_lines = len([x for x in self.routes.values() if x.mode == 'tram']) - self.found_bus_lines = len([x for x in self.routes.values() if x.mode == 'bus']) - self.found_trolleybus_lines = len([x for x in self.routes.values() - if x.mode == 'trolleybus']) - self.found_other_lines = len([x for x in self.routes.values() - if x.mode not in ('bus', 'trolleybus', 'tram')]) + self.found_tram_lines = len( + [x for x in self.routes.values() if x.mode == 'tram'] + ) + self.found_bus_lines = len( + [x for x in self.routes.values() if x.mode == 'bus'] + ) + self.found_trolleybus_lines = len( + [x for x in self.routes.values() if x.mode == 'trolleybus'] + ) + self.found_other_lines = len( + [ + x + for x in self.routes.values() + if x.mode not in ('bus', 'trolleybus', 'tram') + ] + ) if self.found_tram_lines != self.num_tram_lines: - self.error_if(self.found_tram_lines == 0, 'Found {} tram lines, expected {}'.format( - self.found_tram_lines, self.num_tram_lines)) + self.error_if( + self.found_tram_lines == 0, + 'Found {} tram lines, expected {}'.format( + self.found_tram_lines, self.num_tram_lines + ), + ) def validate(self): networks = Counter() @@ -1419,8 +1763,11 @@ class City: self.found_stations += len(route_stations) if unused_stations: self.unused_stations = len(unused_stations) - self.warn('{} unused stations: {}'.format( - self.unused_stations, format_elid_list(unused_stations))) + self.warn( + '{} unused stations: {}'.format( + self.unused_stations, format_elid_list(unused_stations) + ) + ) self.count_unused_entrances() self.found_interchanges = len(self.transfers) @@ -1431,21 +1778,37 @@ class City: if self.found_stations != self.num_stations: msg = 'Found {} stations in routes, expected {}'.format( - self.found_stations, self.num_stations) - self.error_if(not (0 <= - (self.num_stations - self.found_stations) / self.num_stations <= - ALLOWED_STATIONS_MISMATCH), msg) + self.found_stations, self.num_stations + ) + self.error_if( + not ( + 0 + <= (self.num_stations - self.found_stations) + / self.num_stations + <= ALLOWED_STATIONS_MISMATCH + ), + msg, + ) if self.found_interchanges != self.num_interchanges: msg = 'Found {} interchanges, expected {}'.format( - self.found_interchanges, self.num_interchanges) - self.error_if(self.num_interchanges != 0 and not - ((self.num_interchanges - self.found_interchanges) / - self.num_interchanges <= ALLOWED_TRANSFERS_MISMATCH), msg) + self.found_interchanges, self.num_interchanges + ) + self.error_if( + self.num_interchanges != 0 + and not ( + (self.num_interchanges - self.found_interchanges) + / self.num_interchanges + <= ALLOWED_TRANSFERS_MISMATCH + ), + msg, + ) self.found_networks = len(networks) if len(networks) > max(1, len(self.networks)): - n_str = '; '.join(['{} ({})'.format(k, v) for k, v in networks.items()]) + n_str = '; '.join( + ['{} ({})'.format(k, v) for k, v in networks.items()] + ) self.warn('More than one network: {}'.format(n_str)) @@ -1453,8 +1816,11 @@ def find_transfers(elements, cities): transfers = [] stop_area_groups = [] for el in elements: - if (el['type'] == 'relation' and 'members' in el and - el.get('tags', {}).get('public_transport') == 'stop_area_group'): + if ( + el['type'] == 'relation' + and 'members' in el + and el.get('tags', {}).get('public_transport') == 'stop_area_group' + ): stop_area_groups.append(el) # StopArea.id uniquely identifies a StopArea. @@ -1473,7 +1839,9 @@ def find_transfers(elements, cities): k = el_id(m) if k not in stop_area_ids: continue - transfer.update(stop_area_objects[sa_id] for sa_id in stop_area_ids[k]) + transfer.update( + stop_area_objects[sa_id] for sa_id in stop_area_ids[k] + ) if len(transfer) > 1: transfers.append(transfer) return transfers @@ -1483,22 +1851,41 @@ def get_unused_entrances_geojson(elements): global used_entrances features = [] for el in elements: - if (el['type'] == 'node' and 'tags' in el and - el['tags'].get('railway') == 'subway_entrance'): + if ( + el['type'] == 'node' + and 'tags' in el + and el['tags'].get('railway') == 'subway_entrance' + ): if el_id(el) not in used_entrances: geometry = {'type': 'Point', 'coordinates': el_center(el)} - properties = {k: v for k, v in el['tags'].items() - if k not in ('railway', 'entrance')} - features.append({'type': 'Feature', 'geometry': geometry, 'properties': properties}) + properties = { + k: v + for k, v in el['tags'].items() + if k not in ('railway', 'entrance') + } + features.append( + { + 'type': 'Feature', + 'geometry': geometry, + 'properties': properties, + } + ) return {'type': 'FeatureCollection', 'features': features} def download_cities(overground=False): - url = 'https://docs.google.com/spreadsheets/d/{}/export?format=csv{}'.format( - SPREADSHEET_ID, '&gid=1881416409' if overground else '') + url = ( + 'https://docs.google.com/spreadsheets/d/{}/export?format=csv{}'.format( + SPREADSHEET_ID, '&gid=1881416409' if overground else '' + ) + ) response = urllib.request.urlopen(url) if response.getcode() != 200: - raise Exception('Failed to download cities spreadsheet: HTTP {}'.format(response.getcode())) + raise Exception( + 'Failed to download cities spreadsheet: HTTP {}'.format( + response.getcode() + ) + ) data = response.read().decode('utf-8') r = csv.reader(data.splitlines()) next(r) # skipping the header @@ -1508,6 +1895,8 @@ def download_cities(overground=False): if len(row) > 8 and row[8]: cities.append(City(row, overground)) if row[0].strip() in names: - logging.warning('Duplicate city name in the google spreadsheet: %s', row[0]) + logging.warning( + 'Duplicate city name in the google spreadsheet: %s', row[0] + ) names.add(row[0].strip()) return cities diff --git a/validation_to_html.py b/validation_to_html.py index 75fab0a..7790fd4 100755 --- a/validation_to_html.py +++ b/validation_to_html.py @@ -17,7 +17,7 @@ class CityData: 'good_cities': 0, 'total_cities': 1 if city else 0, 'num_errors': 0, - 'num_warnings': 0 + 'num_warnings': 0, } self.slug = None if city: @@ -51,18 +51,34 @@ class CityData: return '1' if v1 == v2 else '0' for k in self.data: - s = s.replace('{'+k+'}', str(self.data[k])) + s = s.replace('{' + k + '}', str(self.data[k])) s = s.replace('{slug}', self.slug or '') - for k in ('subwayl', 'lightrl', 'stations', 'transfers', 'busl', - 'trolleybusl', 'traml', 'otherl'): - if k+'_expected' in self.data: - s = s.replace('{='+k+'}', - test_eq(self.data[k+'_found'], self.data[k+'_expected'])) - s = s.replace('{=cities}', - test_eq(self.data['good_cities'], self.data['total_cities'])) - s = s.replace('{=entrances}', test_eq(self.data['unused_entrances'], 0)) + for k in ( + 'subwayl', + 'lightrl', + 'stations', + 'transfers', + 'busl', + 'trolleybusl', + 'traml', + 'otherl', + ): + if k + '_expected' in self.data: + s = s.replace( + '{=' + k + '}', + test_eq( + self.data[k + '_found'], self.data[k + '_expected'] + ), + ) + s = s.replace( + '{=cities}', + test_eq(self.data['good_cities'], self.data['total_cities']), + ) + s = s.replace( + '{=entrances}', test_eq(self.data['unused_entrances'], 0) + ) for k in ('errors', 'warnings'): - s = s.replace('{='+k+'}', test_eq(self.data['num_'+k], 0)) + s = s.replace('{=' + k + '}', test_eq(self.data['num_' + k], 0)) return s @@ -72,10 +88,19 @@ def tmpl(s, data=None, **kwargs): if kwargs: for k, v in kwargs.items(): if v is not None: - s = s.replace('{'+k+'}', str(v)) - s = re.sub(r'\{\?'+k+r'\}(.+?)\{end\}', r'\1' if v else '', s, flags=re.DOTALL) + s = s.replace('{' + k + '}', str(v)) + s = re.sub( + r'\{\?' + k + r'\}(.+?)\{end\}', + r'\1' if v else '', + s, + flags=re.DOTALL, + ) s = s.replace('{date}', date) - google_url = 'https://docs.google.com/spreadsheets/d/{}/edit?usp=sharing'.format(SPREADSHEET_ID) + google_url = ( + 'https://docs.google.com/spreadsheets/d/{}/edit?usp=sharing'.format( + SPREADSHEET_ID + ) + ) s = s.replace('{google}', google_url) return s @@ -88,13 +113,18 @@ RE_COORDS = re.compile(r'\((-?\d+\.\d+), (-?\d+\.\d+)\)') def osm_links(s): """Converts object mentions to HTML links.""" + def link(m): return '{}'.format( - EXPAND_OSM_TYPE[m.group(1)[0]], m.group(2), m.group(0)) + EXPAND_OSM_TYPE[m.group(1)[0]], m.group(2), m.group(0) + ) + s = RE_SHORT.sub(link, s) s = RE_FULL.sub(link, s) s = RE_COORDS.sub( - r'(pos)', s) + r'(pos)', + s, + ) return s @@ -104,7 +134,9 @@ def esc(s): if len(sys.argv) < 2: print('Reads a log from subway validator and prepares HTML files.') - print('Usage: {} []'.format(sys.argv[0])) + print( + 'Usage: {} []'.format(sys.argv[0]) + ) sys.exit(1) with open(sys.argv[1], 'r', encoding='utf-8') as f: @@ -131,27 +163,67 @@ for continent in sorted(continents.keys()): content = '' for country in sorted(c_by_c[continent]): country_file_name = country.lower().replace(' ', '-') + '.html' - content += tmpl(INDEX_COUNTRY, countries[country], file=country_file_name, - country=country, continent=continent) - country_file = open(os.path.join(path, country_file_name), 'w', encoding='utf-8') - country_file.write(tmpl(COUNTRY_HEADER, country=country, continent=continent, - overground=overground, subways=not overground)) + content += tmpl( + INDEX_COUNTRY, + countries[country], + file=country_file_name, + country=country, + continent=continent, + ) + country_file = open( + os.path.join(path, country_file_name), 'w', encoding='utf-8' + ) + country_file.write( + tmpl( + COUNTRY_HEADER, + country=country, + continent=continent, + overground=overground, + subways=not overground, + ) + ) for name, city in sorted(data.items()): if city.country == country: file_base = os.path.join(path, city.slug) - yaml_file = city.slug + '.yaml' if os.path.exists(file_base + '.yaml') else None - json_file = city.slug + '.geojson' if os.path.exists( - file_base + '.geojson') else None + yaml_file = ( + city.slug + '.yaml' + if os.path.exists(file_base + '.yaml') + else None + ) + json_file = ( + city.slug + '.geojson' + if os.path.exists(file_base + '.geojson') + else None + ) e = '
'.join([osm_links(esc(e)) for e in city.errors]) w = '
'.join([osm_links(esc(w)) for w in city.warnings]) - country_file.write(tmpl(COUNTRY_CITY, city, - city=name, country=country, continent=continent, - yaml=yaml_file, json=json_file, subways=not overground, - errors=e, warnings=w, overground=overground)) - country_file.write(tmpl(COUNTRY_FOOTER, country=country, continent=continent)) + country_file.write( + tmpl( + COUNTRY_CITY, + city, + city=name, + country=country, + continent=continent, + yaml=yaml_file, + json=json_file, + subways=not overground, + errors=e, + warnings=w, + overground=overground, + ) + ) + country_file.write( + tmpl(COUNTRY_FOOTER, country=country, continent=continent) + ) country_file.close() - index.write(tmpl(INDEX_CONTINENT, continents[continent], - content=content, continent=continent)) + index.write( + tmpl( + INDEX_CONTINENT, + continents[continent], + content=content, + continent=continent, + ) + ) index.write(tmpl(INDEX_FOOTER)) index.close() -- 2.45.3 From 90597822f3b394ff241b10f465357f6389c7e802 Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Fri, 10 Sep 2021 14:19:53 +0300 Subject: [PATCH 09/17] Add requirements.txt file for project, to be able to use virtual environments --- requirements.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dab0d26 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Flask==2.0.1 +kdtree==0.16 +lxml==4.6.3 +Shapely==1.7.1 +## The following requirements were added by pip freeze: +click==8.0.1 +itsdangerous==2.0.1 +Jinja2==3.0.1 +MarkupSafe==2.0.1 +Werkzeug==2.0.1 -- 2.45.3 From 4f8f7416f32efa0245ef4242f8bcc2e366f7d061 Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Tue, 28 Sep 2021 18:32:47 +0300 Subject: [PATCH 10/17] Various improvements - Introduce PLANET_UPDATE_SERVER environment variable to allow overriding of default https://planet.openstreetmap.org/replication/ update source. - Explicitly specify path to temp files for osmupdate. - Improve error handling in case when osmupdate fails: capture output and echo them (otherwise bash exits due to `set -e`) - check_osmctools before using osmtools. This fixes error if both SKIP_PLANET_UPDATE and PLANET_METRO are specified. --- scripts/process_subways.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index b0d5f3b..4780678 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -31,6 +31,7 @@ Environment variable reference: - PLANET: path to a local or remote o5m or pbf source file (the entire planet or an extract) - PLANET_METRO: path to a local o5m file with extract of cities having metro It's used instead of \$PLANET if exists otherwise it's created first + - PLANET_UPDATE_SERVER: server to get replication data from. Defaults to https://planet.openstreetmap.org/replication/ - CITY: name of a city/country to process - BBOX: bounding box of an extract; x1,y1,x2,y2. Has precedence over \$POLY - POLY: *.poly file with [multi]polygon comprising cities with metro @@ -185,11 +186,15 @@ fi if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then check_osmctools check_poly + PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/} PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")" + mkdir -p $TMPDIR/osmupdate_temp/ pushd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \ ${POLY:+"-B=$POLY"} "$PLANET_METRO_ABS" \ - "$PLANET_METRO_ABS.new.o5m" 2>&1) + --base-url=$PLANET_UPDATE_SERVER \ + --tempfiles=$TMPDIR/osmupdate_temp/temp \ + "$PLANET_METRO_ABS.new.o5m" 2>&1 || :) if [ -n "$OSMUPDATE_ERRORS" ]; then echo "osmupdate failed: $OSMUPDATE_ERRORS" exit 7 @@ -201,6 +206,7 @@ fi # Filtering planet-metro if [ -n "${NEED_FILTER-}" ]; then + check_osmctools QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group" QNODES="railway=station station=subway =light_rail =monorail railway=subway_entrance subway=yes light_rail=yes monorail=yes train=yes" "$OSMCTOOLS/osmfilter" "$PLANET_METRO" \ -- 2.45.3 From dd833f6639c9a76d14cf6fe676393100ca985f36 Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Wed, 29 Sep 2021 19:45:14 +0300 Subject: [PATCH 11/17] Explicitly specify osmconvert tempfiles location --- scripts/process_subways.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 4780678..7d8010b 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -171,7 +171,9 @@ if [ -n "${NEED_FILTER-}" ]; then exit 6 fi + mkdir -p $TMPDIR/osmconvert_temp/ "$OSMCTOOLS"/osmconvert "$PLANET" \ + -t=$TMPDIR/osmconvert_temp/temp \ ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} -o="$PLANET_METRO" fi fi -- 2.45.3 From 03e3333ec9b16ea2ff5f7e74f5f5809de8c4fa66 Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Thu, 30 Sep 2021 11:50:33 +0300 Subject: [PATCH 12/17] Workaround osmtools bugs with tempfiles - Osmupdate does not pass tempfiles settings to osmconvert call, so ensure that current working dir is writable before launching osmupdate. - Explicitly specify tempfiles location for osmfilter call. --- scripts/process_subways.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 7d8010b..a92c034 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -191,8 +191,9 @@ if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/} PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")" mkdir -p $TMPDIR/osmupdate_temp/ - pushd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory - OSMUPDATE_ERRORS=$(./osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \ + pushd $TMPDIR/osmupdate_temp/ + export PATH="$PATH:$OSMCTOOLS" + OSMUPDATE_ERRORS=$(osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \ ${POLY:+"-B=$POLY"} "$PLANET_METRO_ABS" \ --base-url=$PLANET_UPDATE_SERVER \ --tempfiles=$TMPDIR/osmupdate_temp/temp \ @@ -209,6 +210,7 @@ fi if [ -n "${NEED_FILTER-}" ]; then check_osmctools + mkdir -p $TMPDIR/osmfilter_temp/ QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group" QNODES="railway=station station=subway =light_rail =monorail railway=subway_entrance subway=yes light_rail=yes monorail=yes train=yes" "$OSMCTOOLS/osmfilter" "$PLANET_METRO" \ @@ -216,6 +218,7 @@ if [ -n "${NEED_FILTER-}" ]; then --keep-relations="$QRELATIONS" \ --keep-nodes="$QNODES" \ --drop-author \ + -t=$TMPDIR/osmfilter_temp/temp \ -o="$FILTERED_DATA" fi -- 2.45.3 From 2cb5e8469466b1c7980e3dcc11de41e1a28b4989 Mon Sep 17 00:00:00 2001 From: Claudius Date: Wed, 6 Oct 2021 12:03:36 +0200 Subject: [PATCH 13/17] Update v2h_templates.py Tweaks to the HTML output template: - Sticky footer so update timestamp always visible without scrolling - Slight copy update to clarify that networks are shown behind the map link - table row hover state now uses hue-rotate to ensure enough contrast as compared to whole line blue previously --- v2h_templates.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/v2h_templates.py b/v2h_templates.py index 0f0237b..300a62e 100644 --- a/v2h_templates.py +++ b/v2h_templates.py @@ -6,6 +6,10 @@ STYLE = ''' body { font-family: sans-serif; font-size: 12pt; + margin: 0px; +} +main { + margin: 10px; } th { font-size: 10pt; @@ -37,7 +41,14 @@ table { max-width: 900px; } tr:hover td:nth-child(n+2) { - background: lightblue; + filter: hue-rotate(-50deg); +} +footer { + background: white; + border-top: 1px solid grey; + bottom: 0px; + padding: 10px; + position: sticky; } ''' @@ -51,9 +62,10 @@ INDEX_HEADER = ''' (s) +

Subway Validation Results

Total good metro networks: {good_cities} of {total_cities}.

-

View on the map

+

View networks on a map

'''.replace('(s)', STYLE) @@ -99,10 +111,11 @@ INDEX_COUNTRY = ''' INDEX_FOOTER = '''
-

Produced by Subway Preprocessor on {date}. +

+ ''' @@ -116,6 +129,7 @@ COUNTRY_HEADER = ''' (s) +

Subway Validation Results for {country}

Return to the countries list.

@@ -168,7 +182,8 @@ COUNTRY_CITY = ''' COUNTRY_FOOTER = '''
-

Produced by Subway Preprocessor on {date}.

+
+ ''' -- 2.45.3 From 0a1115f4ecda76f49907cf500ea29ff230fb2766 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Fri, 15 Oct 2021 16:32:07 +0300 Subject: [PATCH 14/17] Make three levels of severity for validation errors --- subway_structure.py | 142 ++++++++++++++++++++++-------------------- v2h_templates.py | 16 ++++- validation_to_html.py | 15 +++-- 3 files changed, 100 insertions(+), 73 deletions(-) diff --git a/subway_structure.py b/subway_structure.py index 222a9db..83d8af9 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -387,7 +387,7 @@ class StopArea: self.exits.add(k) elif StopArea.is_track(m_el): if not warned_about_tracks: - city.error( + city.warn( 'Tracks in a stop_area relation', stop_area ) warned_about_tracks = True @@ -415,12 +415,12 @@ class StopArea: self.exits.add(c_id) if self.exits and not self.entrances: - city.error( + city.warn( 'Only exits for a station, no entrances', stop_area or station.element, ) if self.entrances and not self.exits: - city.error('No exits for a station', stop_area or station.element) + city.warn('No exits for a station', stop_area or station.element) for el in self.get_elements(): self.centers[el] = el_center(city.elements[el]) @@ -502,7 +502,7 @@ class RouteStop: elif Station.is_station(el, city.modes): if el['type'] != 'node': - city.warn('Station in route is not a node', el) + city.notice('Station in route is not a node', el) if not self.seen_stop and not self.seen_platform: self.stop = el_center(el) @@ -521,9 +521,6 @@ class RouteStop: if not self.seen_stop: self.stop = el_center(el) - else: - city.error('Not a stop or platform in a route relation', el) - multiple_check = False actual_role = RouteStop.get_actual_role(el, role, city.modes) if actual_role == 'platform': @@ -548,8 +545,8 @@ class RouteStop: multiple_check = self.seen_stop self.seen_stop = True if multiple_check: - city.error_if( - actual_role == 'stop', + log_function = city.error if actual_role == 'stop' else city.notice + log_function( 'Multiple {}s for a station "{}" ({}) in a route relation'.format( actual_role, el['tags'].get('name', ''), el_id(el) ), @@ -713,7 +710,7 @@ class Route: # railway tracks between them. Put these on tracks. d = round(distance(route_stop.stop, projected_point)) if d > MAX_DISTANCE_STOP_TO_LINE: - self.city.warn( + self.city.notice( 'Stop "{}" {} is {} meters from the tracks'.format( route_stop.stoparea.name, route_stop.stop, d ), @@ -757,7 +754,7 @@ class Route: self.element = relation self.id = el_id(relation) if 'ref' not in relation['tags'] and 'ref' not in master_tags: - city.warn('Missing ref on a route', relation) + city.notice('Missing ref on a route', relation) self.ref = relation['tags'].get( 'ref', master_tags.get('ref', relation['tags'].get('name', None)) ) @@ -768,7 +765,7 @@ class Route: and 'colour' not in master_tags and self.mode != 'tram' ): - city.warn('Missing colour on a route', relation) + city.notice('Missing colour on a route', relation) try: self.colour = normalize_colour( relation['tags'].get('colour', master_tags.get('colour', None)) @@ -790,7 +787,7 @@ class Route: relation['tags'] ) or Route.get_interval(master_tags) if relation['tags'].get('public_transport:version') == '1': - city.error( + city.warn( 'Public transport version is 1, which means the route ' 'is an unsorted pile of objects', relation, @@ -804,16 +801,14 @@ class Route: ): # usually, extending BBOX for the city is needed self.tracks = [] for n in filter(lambda x: x not in city.elements, tracks): - city.error( + city.warn( 'The dataset is missing the railway tracks node {}'.format( n ), relation, ) break - check_stop_positions = ( - len(line_nodes) > 50 - ) # arbitrary number, of course + self.stops = [] # List of RouteStop stations = set() # temporary for recording stations seen_stops = False @@ -905,7 +900,7 @@ class Route: seen_stops |= stop.seen_stop or stop.seen_station seen_platforms |= stop.seen_platform - if check_stop_positions and StopArea.is_stop(el): + if StopArea.is_stop(el): if k not in line_nodes: city.warn( 'Stop position "{}" ({}) is not on tracks'.format( @@ -925,13 +920,13 @@ class Route: continue el = city.elements[k] if 'tags' not in el: - city.error('Untagged object in a route', relation) + city.error('Untagged object {} in a route'.format(k), relation) continue is_under_construction = False for ck in CONSTRUCTION_KEYS: if ck in el['tags']: - city.error( + city.warn( 'An under construction {} {} in route. Consider ' 'setting \'inactive\' role or removing construction attributes'.format( m['role'] or 'feature', k @@ -964,7 +959,7 @@ class Route: relation, ) elif not StopArea.is_track(el): - city.error( + city.warn( 'Unknown member type for {} {} in route'.format( m['type'], m['ref'] ), @@ -1093,7 +1088,7 @@ class Route: resort_success = self.try_resort_stops() if resort_success: for msg in disorder_warnings: - self.city.warn(msg, self.element) + self.city.notice(msg, self.element) for msg in disorder_errors: self.city.warn( "Fixed with recovery data: " + msg, self.element @@ -1101,7 +1096,7 @@ class Route: if not resort_success: for msg in disorder_warnings: - self.city.warn(msg, self.element) + self.city.notice(msg, self.element) for msg in disorder_errors: self.city.error(msg, self.element) @@ -1255,7 +1250,7 @@ class RouteMaster: if not self.colour: self.colour = route.colour elif route.colour and route.colour != self.colour: - city.warn( + city.notice( 'Route "{}" has different colour from master "{}"'.format( route.colour, self.colour ), @@ -1265,7 +1260,7 @@ class RouteMaster: if not self.infill: self.infill = route.infill elif route.infill and route.infill != self.infill: - city.warn( + city.notice( 'Route "{}" has different infill colour from master "{}"'.format( route.infill, self.infill ), @@ -1275,7 +1270,7 @@ class RouteMaster: if not self.ref: self.ref = route.ref elif route.ref != self.ref: - city.warn( + city.notice( 'Route "{}" has different ref from master "{}"'.format( route.ref, self.ref ), @@ -1343,6 +1338,7 @@ class City: def __init__(self, row, overground=False): self.errors = [] self.warnings = [] + self.notices = [] self.name = row[1] self.country = row[2] self.continent = row[3] @@ -1396,7 +1392,8 @@ class City: self.stops_and_platforms = set() # Set of stops and platforms el_id self.recovery_data = None - def log_message(self, message, el): + @staticmethod + def log_message(message, el): if el: tags = el.get('tags', {}) message += ' ({} {}, "{}")'.format( @@ -1406,20 +1403,23 @@ class City: ) return message + def notice(self, message, el=None): + """This type of message may point to a potential problem.""" + msg = City.log_message(message, el) + self.notices.append(msg) + def warn(self, message, el=None): - msg = self.log_message(message, el) + """A warning is definitely a problem but is doesn't prevent + from building a routing file and doesn't invalidate the city. + """ + msg = City.log_message(message, el) self.warnings.append(msg) def error(self, message, el=None): - msg = self.log_message(message, el) + """Error if a critical problem that invalidates the city""" + msg = City.log_message(message, el) self.errors.append(msg) - def error_if(self, is_error, message, el=None): - if is_error: - self.error(message, el) - else: - self.warn(message, el) - def contains(self, el): center = el_center(el) if center: @@ -1461,7 +1461,7 @@ class City: # the sag does - near the city bbox boundary continue if 'tags' not in el: - self.error( + self.warn( 'An untagged object {} in a stop_area_group'.format(k), sag ) continue @@ -1475,7 +1475,13 @@ class City: stoparea = self.stations[k][0] transfer.add(stoparea) if stoparea.transfer: - self.error( + # TODO: properly process such cases. + # Counterexample 1: Paris, + # Châtelet subway station <-> + # "Châtelet - Les Halles" railway station <-> + # Les Halles subway station + # Counterexample 2: Saint-Petersburg, transfers Витебский вокзал <-> Пушкинская <-> Звенигородская + self.warn( 'Stop area {} belongs to multiple interchanges'.format( k ) @@ -1494,7 +1500,7 @@ class City: el['type'] == 'relation' and el['tags'].get('type') != 'multipolygon' ): - self.error( + self.warn( "A railway station cannot be a relation of type '{}'".format( el['tags'].get('type') ), @@ -1519,9 +1525,9 @@ class City: # Check that stops and platforms belong to single stop_area for sp in station.stops | station.platforms: if sp in self.stops_and_platforms: - self.warn( + self.notice( 'A stop or a platform {} belongs to multiple ' - 'stations, might be correct'.format(sp) + 'stop areas, might be correct'.format(sp) ) else: self.stops_and_platforms.add(sp) @@ -1621,6 +1627,7 @@ class City: ) result['warnings'] = self.warnings result['errors'] = self.errors + result['notices'] = self.notices return result def count_unused_entrances(self): @@ -1652,13 +1659,13 @@ class City: self.unused_entrances = len(unused) self.entrances_not_in_stop_areas = len(not_in_sa) if unused: - self.warn( - 'Found {} entrances not used in routes or stop_areas: {}'.format( + self.notice( + '{} subway entrances are not connected to a station: {}'.format( len(unused), format_elid_list(unused) ) ) if not_in_sa: - self.warn( + self.notice( '{} subway entrances are not in stop_area relations: {}'.format( len(not_in_sa), format_elid_list(not_in_sa) ) @@ -1694,8 +1701,8 @@ class City: 'if it is under construction'.format(rmaster.id) ) elif len(variants) == 1: - self.error_if( - not rmaster.best.is_circular, + log_function = self.error if not rmaster.best.is_circular else self.notice + log_function( 'Only one route in route_master. ' 'Please check if it needs a return route', rmaster.best.element, @@ -1703,7 +1710,7 @@ class City: else: for t, rel in variants.items(): if t not in have_return: - self.warn('Route does not have a return direction', rel) + self.notice('Route does not have a return direction', rel) def validate_lines(self): self.found_light_lines = len( @@ -1741,8 +1748,7 @@ class City: ] ) if self.found_tram_lines != self.num_tram_lines: - self.error_if( - self.found_tram_lines == 0, + self.error( 'Found {} tram lines, expected {}'.format( self.found_tram_lines, self.num_tram_lines ), @@ -1763,7 +1769,7 @@ class City: self.found_stations += len(route_stations) if unused_stations: self.unused_stations = len(unused_stations) - self.warn( + self.notice( '{} unused stations: {}'.format( self.unused_stations, format_elid_list(unused_stations) ) @@ -1780,36 +1786,40 @@ class City: msg = 'Found {} stations in routes, expected {}'.format( self.found_stations, self.num_stations ) - self.error_if( - not ( - 0 - <= (self.num_stations - self.found_stations) - / self.num_stations - <= ALLOWED_STATIONS_MISMATCH - ), - msg, + log_function = ( + self.error + if not ( + 0 + <= (self.num_stations - self.found_stations) + / self.num_stations + <= ALLOWED_STATIONS_MISMATCH + ) + else self.warn ) + log_function(msg) if self.found_interchanges != self.num_interchanges: msg = 'Found {} interchanges, expected {}'.format( self.found_interchanges, self.num_interchanges ) - self.error_if( - self.num_interchanges != 0 - and not ( - (self.num_interchanges - self.found_interchanges) - / self.num_interchanges - <= ALLOWED_TRANSFERS_MISMATCH - ), - msg, + log_function = ( + self.error + if self.num_interchanges != 0 + and not ( + (self.num_interchanges - self.found_interchanges) + / self.num_interchanges + <= ALLOWED_TRANSFERS_MISMATCH + ) + else self.warn ) + log_function(msg) self.found_networks = len(networks) if len(networks) > max(1, len(self.networks)): n_str = '; '.join( ['{} ({})'.format(k, v) for k, v in networks.items()] ) - self.warn('More than one network: {}'.format(n_str)) + self.notice('More than one network: {}'.format(n_str)) def find_transfers(elements, cities): diff --git a/v2h_templates.py b/v2h_templates.py index 0f0237b..5736212 100644 --- a/v2h_templates.py +++ b/v2h_templates.py @@ -12,10 +12,15 @@ th { } .errors { font-size: 10pt; - color: darkred; + color: red; margin-bottom: 1em; } .warnings { + font-size: 10pt; + color: saddlebrown; + margin-bottom: 1em; +} +.notices { font-size: 10pt; color: darkblue; margin-bottom: 1em; @@ -69,6 +74,7 @@ INDEX_CONTINENT = ''' Interchanges Errors Warnings +Notices {continent} @@ -79,6 +85,7 @@ INDEX_CONTINENT = ''' {transfers_found} / {transfers_expected} {num_errors} {num_warnings} +{num_notices} {content} ''' @@ -94,6 +101,7 @@ INDEX_COUNTRY = ''' {transfers_found} / {transfers_expected} {num_errors} {num_warnings} +{num_notices} ''' @@ -160,9 +168,13 @@ COUNTRY_CITY = '''
{errors} -
+
+
{warnings}
+
+{notices} +
''' diff --git a/validation_to_html.py b/validation_to_html.py index 7790fd4..f4468de 100755 --- a/validation_to_html.py +++ b/validation_to_html.py @@ -18,6 +18,7 @@ class CityData: 'total_cities': 1 if city else 0, 'num_errors': 0, 'num_warnings': 0, + 'num_notices': 0 } self.slug = None if city: @@ -26,10 +27,12 @@ class CityData: self.continent = city['continent'] self.errors = city['errors'] self.warnings = city['warnings'] + self.notices = city['notices'] if not self.errors: self.data['good_cities'] = 1 self.data['num_errors'] = len(self.errors) self.data['num_warnings'] = len(self.warnings) + self.data['num_notices'] = len(self.notices) for k, v in city.items(): if 'found' in k or 'expected' in k or 'unused' in k: self.data[k] = v @@ -77,7 +80,7 @@ class CityData: s = s.replace( '{=entrances}', test_eq(self.data['unused_entrances'], 0) ) - for k in ('errors', 'warnings'): + for k in ('errors', 'warnings', 'notices'): s = s.replace('{=' + k + '}', test_eq(self.data['num_' + k], 0)) return s @@ -195,8 +198,9 @@ for continent in sorted(continents.keys()): if os.path.exists(file_base + '.geojson') else None ) - e = '
'.join([osm_links(esc(e)) for e in city.errors]) - w = '
'.join([osm_links(esc(w)) for w in city.warnings]) + errors = '
'.join([osm_links(esc(e)) for e in city.errors]) + warnings = '
'.join([osm_links(esc(w)) for w in city.warnings]) + notices = '
'.join([osm_links(esc(n)) for n in city.notices]) country_file.write( tmpl( COUNTRY_CITY, @@ -207,8 +211,9 @@ for continent in sorted(continents.keys()): yaml=yaml_file, json=json_file, subways=not overground, - errors=e, - warnings=w, + errors=errors, + warnings=warnings, + notices=notices, overground=overground, ) ) -- 2.45.3 From 0bc8f779ae79ddf53e3811b90181ea650436481d Mon Sep 17 00:00:00 2001 From: Claudius Date: Fri, 22 Oct 2021 11:16:11 +0200 Subject: [PATCH 15/17] More user friendly UX for severity level messaging More user friendly severity level messaging Included the changes I mentioned in [my earlier comment](https://github.com/alexey-zakharenkov/subways/pull/14#issuecomment-948391863) to: - Add section headers to the different severity level messages - Change the errors red colour to a accessible one - Changed copy of entrance errors from "e:" to "ent:" - Shortened an error message slightly to reduce possibility of it wrapping into a new line and have the "Under construction" keyword at the beginning of the sentence CSS property width=fit-content is not supported in Firefox; shorten tooltip text; exclude empty validation message during HTML genertion --- subway_structure.py | 2 +- v2h_templates.py | 79 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/subway_structure.py b/subway_structure.py index 83d8af9..280597b 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -927,7 +927,7 @@ class Route: for ck in CONSTRUCTION_KEYS: if ck in el['tags']: city.warn( - 'An under construction {} {} in route. Consider ' + 'Under construction {} {} in route. Consider ' 'setting \'inactive\' role or removing construction attributes'.format( m['role'] or 'feature', k ), diff --git a/v2h_templates.py b/v2h_templates.py index 5736212..0eddc02 100644 --- a/v2h_templates.py +++ b/v2h_templates.py @@ -7,23 +7,24 @@ body { font-family: sans-serif; font-size: 12pt; } +main { + margin: 0 auto; + max-width: 900px; +} th { font-size: 10pt; } .errors { font-size: 10pt; - color: red; - margin-bottom: 1em; + color: #ED0000; } .warnings { font-size: 10pt; color: saddlebrown; - margin-bottom: 1em; } .notices { font-size: 10pt; color: darkblue; - margin-bottom: 1em; } .bold { font-weight: bold; @@ -43,6 +44,58 @@ table { } tr:hover td:nth-child(n+2) { background: lightblue; + box-shadow: 0px 0px 5px lightblue; +} +td { + border-radius: 2px; +} +td div+div { + margin-top: 0.8em; +} +.tooltip { + font-weight: bold; + position: relative; + text-align: left; +} +.tooltip div { + display: inline-block; + width: 19px; +} +.tooltip:before { + content: attr(data-text); + position: absolute; + top: 100%; + transform: translateX(-50%); + margin-top: 14px; + width: 200px; + padding: 10px; + border-radius: 10px; + background: lightblue; + color: black; + text-align: center; + opacity: 0; + transition: .3s opacity; + visibility: hidden; + z-index: 10 +} +.tooltip:after { + content: ""; + position: absolute; + margin-top: -5px; + top: 100%; + transform: translateX(-50%); + border: 10px solid #000; + border-color: transparent transparent lightblue transparent; + visibility: hidden; + opacity: 0; + transition: .3s opacity +} +.tooltip:hover { + text-decoration: none +} +.tooltip:hover:before,.tooltip:hover:after { + opacity: 1; + visibility: visible } ''' @@ -56,6 +109,7 @@ INDEX_HEADER = ''' (s) +

Subway Validation Results

Total good metro networks: {good_cities} of {total_cities}.

View on the map

@@ -107,6 +161,7 @@ INDEX_COUNTRY = ''' INDEX_FOOTER = ''' +

Produced by Subway Preprocessor on {date}. See this spreadsheet for the reference metro statistics and this wiki page for a list @@ -124,6 +179,7 @@ COUNTRY_HEADER = ''' (s) +

Subway Validation Results for {country}

Return to the countries list.

@@ -163,23 +219,30 @@ COUNTRY_CITY = ''' {end} - + ''' COUNTRY_FOOTER = '''
st: {stations_found} / {stations_expected} int: {transfers_found} / {transfers_expected}e: {unused_entrances}ent: {unused_entrances}
-
+{?errors} +
🛑 Errors
{errors}
-
+{end} +{?warnings} +
⚠️ Warnings
{warnings}
-
+{end} +{?notices} +
ℹ️ Notices
{notices} +{end}
+

Produced by Subway Preprocessor on {date}.

-- 2.45.3 From 76d9d0bbb9f539f7e543856974da6b7b76cf4ca6 Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Wed, 3 Nov 2021 23:30:32 +0300 Subject: [PATCH 16/17] Fix html layout so that tooltips stay inside window viewport --- v2h_templates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2h_templates.py b/v2h_templates.py index eb825f1..bcec823 100644 --- a/v2h_templates.py +++ b/v2h_templates.py @@ -52,8 +52,8 @@ tr:hover td:nth-child(n+2) { td { border-radius: 2px; } -td div+div { - margin-top: 0.8em; +td > div { + margin-bottom: 0.8em; } .tooltip { font-weight: bold; @@ -68,7 +68,7 @@ td div+div { content: attr(data-text); position: absolute; top: 100%; - transform: translateX(-50%); + left: 0; margin-top: 14px; width: 200px; padding: 10px; @@ -86,7 +86,7 @@ td div+div { position: absolute; margin-top: -5px; top: 100%; - transform: translateX(-50%); + left: 30px; border: 10px solid #000; border-color: transparent transparent lightblue transparent; visibility: hidden; -- 2.45.3 From ef5bbab6723f0b62082d085ac29e3f5eace80f1d Mon Sep 17 00:00:00 2001 From: Alexey Zakharenkov Date: Mon, 8 Nov 2021 12:22:57 +0300 Subject: [PATCH 17/17] Return "error_if"-logic for tram lines count --- subway_structure.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subway_structure.py b/subway_structure.py index 280597b..b925bb5 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -1748,7 +1748,8 @@ class City: ] ) if self.found_tram_lines != self.num_tram_lines: - self.error( + log_function = self.error if self.found_tram_lines == 0 else self.notice + log_function( 'Found {} tram lines, expected {}'.format( self.found_tram_lines, self.num_tram_lines ), -- 2.45.3