organicmaps/subways
13
Fork 0
Code Issues Pull requests 1 Projects Releases Wiki Activity Actions
subways/scripts/process_subways.sh
Ilya Zverev fce16592cf FMK
2024-12-09 17:45:39 +02:00

299 lines
10 KiB
Bash
Executable file
Raw Blame History

#!/bin/bash
set -e -u
if [ $# -gt 0 -a \( "${1-}" = "-h" -o "${1-}" = '--help' \) ]; then
cat << EOF
This script updates a planet or an extract, processes metro networks in it
and produces a set of HTML files with validation results.
Usage: $0 [<local/planet.osm.pbf | http://mirror.osm.ru/planet.osm.pbf>]
In more detail, the script does the following:
- If \$PLANET is a remote file, downloads it.
- If \$BBOX variable is set, proceeds with this setting for the planet clipping. Otherwise uses \$POLY:
unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro.
- Makes an extract of the \$PLANET using the *.poly file.
- Updates the extract.
- Filters railway infrastructure from the extract.
- Uses filtered file for validation and generates a bunch of output files.
- Copies results onto remote server, if it is set up.
During this procedure, as many steps are skipped as possible. Namely:
- Generation of metro extract is skipped if \$PLANET_METRO variable is set and the file exists.
- Update with osmupdate is skipped if \$SKIP_PLANET_UPDATE or \$SKIP_FILTERING is set.
- Filtering is skipped if \$SKIP_FILTERING is set and \$FILTERED_DATA is set and the file exists.
Generated files \$POLY, \$PLANET_METRO, \$FILTERED_DATA are deleted if the corresponding
variable is not defined or is null, otherwise they are kept.
The \$PLANET file from remote URL is saved to a tempfile and is removed at the end.
Environment variable reference:
- PLANET: path to a local or remote pbf source file (the entire planet or an extract)
- PLANET_METRO: path to a local pbf file with extract of cities having metro
It's used instead of \$PLANET if exists otherwise it's created first
- PLANET_UPDATE_SERVER: server to get replication data from. Defaults to https://planet.openstreetmap.org/replication/
- CITIES_INFO_URL: http(s) or "file://" URL to a CSV file with reference information about rapid transit systems. A default value is hammered into python code.
- CITY: name of a city/country to process
- BBOX: bounding box of an extract; x1,y1,x2,y2. Has precedence over \$POLY
- POLY: *.poly file with [multi]polygon comprising cities with metro
If neither \$BBOX nor \$POLY is set, then \$POLY is generated
- SKIP_PLANET_UPDATE: skip \$PLANET_METRO file update. Any non-empty string is True
- SKIP_FILTERING: skip filtering railway data. Any non-empty string is True
- FILTERED_DATA: path to filtered data. Defaults to \$TMPDIR/subways.osm
- MAPSME: file name for maps.me json output
- GTFS: file name for GTFS output
- DUMP: directory/file name to dump YAML city data. Do not set to omit dump
- GEOJSON: directory/file name to dump GeoJSON data. Do not set to omit dump
- ELEMENTS_CACHE: file name to elements cache. Allows OSM xml processing phase
- CITY_CACHE: json file with good cities obtained on previous validation runs
- RECOVERY_PATH: file with some data collected at previous validation runs that
may help to recover some simple validation errors
- PYTHON: python 3 executable
- GIT_PULL: set to 1 to update the scripts
- TMPDIR: path to temporary files
- HTML_DIR: target path for generated HTML files
- DUMP_CITY_LIST: file name to save sorted list of cities, with [bad] mark for bad cities
- SERVER: server name and path to upload HTML files (e.g. ilya@osmz.ru:/var/www/)
- SERVER_KEY: rsa key to supply for uploading the files
- REMOVE_HTML: set to 1 to remove \$HTML_DIR after uploading
- QUIET: set to any non-empty value to use WARNING log level in process_subways.py. Default is INFO.
EOF
exit
fi
function activate_venv_at_path() {
path=$1
if [ ! -d "$path/".venv ]; then
"${PYTHON:-python3}" -m venv "$path"/.venv
fi
source "$path"/.venv/bin/activate
if [ -f "$path"/requirements.txt ]; then
pip install --upgrade pip
pip install -r "$path"/requirements.txt
fi
}
function check_osmium() {
if ! which osmium > /dev/null; then
echo "Please install osmium-tool"
exit 1
fi
}
function check_poly() {
# Checks or generates *.poly file covering cities where
# there is a metro; does this only once during script run.
if [ -z "${POLY_CHECKED-}" ]; then
if [ -n "${BBOX-}" ]; then
# If BBOX is set, then exclude POLY at all from processing
POLY=""
else
if [ -z "${POLY-}" ]; then
NEED_TO_REMOVE_POLY=1
fi
if [ -z "${POLY-}" -o ! -f "${POLY-}" ]; then
POLY=${POLY:-$(mktemp "$TMPDIR/all-metro.XXXXXXXX.poly")}
activate_venv_at_path "$SUBWAYS_REPO_PATH/tools/make_poly"
python "$SUBWAYS_REPO_PATH"/tools/make_poly/make_all_metro_poly.py \
${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} > "$POLY"
deactivate
fi
fi
POLY_CHECKED=1
fi
}
# "readlink -f" echoes canonicalized absolute path to a file/directory
SUBWAYS_REPO_PATH="$(readlink -f $(dirname "$0")/..)"
if [ ! -f "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ]; then
echo "Please clone the subways repo to $SUBWAYS_REPO_PATH"
exit 2
fi
# Contains 'subways' dir and is required by the main validator python script
# as well as by some tools
export PYTHONPATH="$SUBWAYS_REPO_PATH"
# Downloading the latest version of the subways script
if [ -n "${GIT_PULL-}" ]; then (
pushd "$SUBWAYS_REPO_PATH"
git pull origin master
popd
) fi
TMPDIR="${TMPDIR:-"$SUBWAYS_REPO_PATH"}"
mkdir -p "$TMPDIR"
if [ -z "${FILTERED_DATA-}" ]; then
FILTERED_DATA="$TMPDIR/subways.osm"
NEED_TO_REMOVE_FILTERED_DATA=1
fi
if [ -z "${SKIP_FILTERING-}" -o ! -f "$FILTERED_DATA" ]; then
NEED_FILTER=1
fi
if [ -n "${NEED_FILTER-}" ]; then
# If $PLANET_METRO file doesn't exist, create it
if [ -n "${PLANET_METRO-}" ]; then
EXT=${PLANET_METRO##*.}
if [ ! "$EXT" = "osm" -a ! "$EXT" == "xml" -a ! "$EXT" = "o5m" ]; then
echo "Only o5m/xml/osm file formats are supported for filtering."
exit 3
fi
fi
if [ ! -f "${PLANET_METRO-}" ]; then
check_osmium
check_poly
PLANET="${PLANET:-${1-}}"
EXT="${PLANET##*.}"
if [ ! "$EXT" = "pbf" -a ! "$EXT" = "o5m" ]; then
echo "Cannot process '$PLANET' planet file."
echo "Only pbf/o5m source planet files are supported."
exit 4
fi
if [ "${PLANET:0:7}" = "http://" -o \
"${PLANET:0:8}" = "https://" -o \
"${PLANET:0:6}" = "ftp://" ]; then
PLANET_TEMP=$(mktemp "$TMPDIR/planet.XXXXXXXX.$EXT")
wget -O "$PLANET_TEMP" "$PLANET"
PLANET="$PLANET_TEMP"
elif [ ! -f "$PLANET" ]; then
echo "Cannot find planet file '$PLANET'";
exit 5
fi
if [ -z "${PLANET_METRO-}" ]; then
PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.osm.pbf")
NEED_TO_REMOVE_PLANET_METRO=1
fi
if [ "$PLANET" = "$PLANET_METRO" ]; then
echo "PLANET_METRO parameter shouldn't point to PLANET."
exit 6
fi
osmium extract "$PLANET" \
${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO"
fi
fi
if [ -n "${PLANET_TEMP-}" ]; then
rm "$PLANET_TEMP"
fi
# Updating the planet-metro file
# If there's no need to filter, then update is also unnecessary
if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then
check_osmium
check_poly
PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/hour/}
PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")"
PLANET_METRO_ABS_NEW="$PLANET_METRO_ABS.new.osm.pbf"
mkdir -p $TMPDIR/osmupdate_temp/
activate_venv_at_path "$SUBWAYS_REPO_PATH/scripts"
OSMUPDATE_ERRORS=$(pyosmium-up-to-date \
"$PLANET_METRO_ABS" \
--server $PLANET_UPDATE_SERVER \
--tmpdir $TMPDIR/osmupdate_temp/temp \
-o "$PLANET_METRO_ABS_NEW" 2>&1 || :)
deactivate
if [ -n "$OSMUPDATE_ERRORS" ]; then
echo "osmupdate failed: $OSMUPDATE_ERRORS"
exit 7
fi
# Since updating adds things outside the area, trim those again.
osmium extract "$PLANET_METRO_ABS_NEW" \
${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO_ABS"
rm -f "$PLANET_METRO_ABS_NEW"
fi
# Filtering planet-metro
if [ -n "${NEED_FILTER-}" ]; then
check_osmium
QRELATIONS="r/route,route_master=subway,light_rail,monorail,train r/public_transport=stop_area,stop_area_group"
QNODES="n/railway=station,subway_entrance,train_station_entrance n/station=subway,light_rail,monorail n/subway=yes n/light_rail=yes n/monorail=yes n/train=yes"
osmium tags-filter "$PLANET_METRO" $QRELATIONS $QNODES -o "$FILTERED_DATA" -O
fi
if [ -n "${NEED_TO_REMOVE_PLANET_METRO-}" ]; then
rm $PLANET_METRO
fi
if [ -n "${NEED_TO_REMOVE_POLY-}" ]; then
rm $POLY
fi
# Running the validation
if [ -n "${DUMP-}" ]; then
mkdir -p "$DUMP"
fi
VALIDATION="$TMPDIR/validation.json"
activate_venv_at_path "$SUBWAYS_REPO_PATH/scripts"
python "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ${QUIET:+-q} \
-x "$FILTERED_DATA" -l "$VALIDATION" \
${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
${MAPSME:+--output-mapsme "$MAPSME"} \
${FMK:+--output-fmk "$FMK"} \
${GTFS:+--output-gtfs "$GTFS"} \
${CITY:+-c "$CITY"} \
${DUMP:+-d "$DUMP"} \
${GEOJSON:+-j "$GEOJSON"} \
${DUMP_CITY_LIST:+--dump-city-list "$DUMP_CITY_LIST"} \
${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} \
${CITY_CACHE:+--cache "$CITY_CACHE"} \
${RECOVERY_PATH:+-r "$RECOVERY_PATH"}
deactivate
if [ -n "${NEED_TO_REMOVE_FILTERED_DATA-}" ]; then
rm "$FILTERED_DATA"
fi
# Preparing HTML files
if [ -z "${HTML_DIR-}" ]; then
HTML_DIR="$SUBWAYS_REPO_PATH/html"
REMOVE_HTML=1
fi
mkdir -p $HTML_DIR
rm -f "$HTML_DIR"/*.html
activate_venv_at_path "$SUBWAYS_REPO_PATH/tools/v2h"
python "$SUBWAYS_REPO_PATH/tools/v2h/validation_to_html.py" \
${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
"$VALIDATION" "$HTML_DIR"
deactivate
# Uploading files to the server
if [ -n "${SERVER-}" ]; then
scp -q ${SERVER_KEY+-i "$SERVER_KEY"} "$HTML_DIR"/* "$SERVER"
if [ -n "${REMOVE_HTML-}" ]; then
rm -r "$HTML_DIR"
fi
fi
Reference in a new issue View git blame Copy permalink