Merge branch 'master' into upstream

Signed-off-by: Alexander Borsuk <170263+biodranik@users.noreply.github.com>

.gitignore

@@ -13,5 +13,3 @@ html/
 *.pyc
 *.txt
 *.zip
-*.osm.pbf
-*.o5m

README.md

@@ -34,7 +34,8 @@ for details. Here is an example of the script usage:
 
 ```bash
 export PLANET=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf
-export PLANET_METRO="$HOME/metro/planet-metro.osm.pbf
+export PLANET_METRO="$HOME/metro/planet-metro.o5m
+export OSMCTOOLS="$HOME/osmctools"
 export TMPDIR="$HOME/metro/tmp"
 export HTML_DIR="$HOME/metro/tmp_html"
 export DUMP="$HTML_DIR"
@@ -59,15 +60,14 @@ if you allow the `scripts/process_subway.py` to fetch data from Overpass API. He
     cd subways_validator
    ```
 3. Configure python environment, e.g.
-    ```bash
-    python3 -m venv scripts/.venv
-    source scripts/.venv/bin/activate
-    pip install -r scripts/requirements.txt
-    ```
-    (this is optional if you only process a single city though.)
+   ```bash
+   python3 -m venv scripts/.venv
+   source scripts/.venv/bin/activate
+   pip install scripts/requirements.txt
+   ```
 4. Execute
     ```bash
-    PYTHONPATH=. python3 scripts/process_subways.py -c "London" \
+    python3 scripts/process_subways.py -c "London" \
         -l validation.log -d London.yaml
     ```
     here
@@ -101,8 +101,8 @@ Expose a directory with static contents via a web-server and put into it:
 Summary information about all metro networks that are monitored is gathered in the
 [Google Spreadsheet](https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k).
 
-Regular updates of validation results are available at
-[this website](https://cdn.organicmaps.app/subway).
+Regular updates of validation results are available at [Organic Maps](https://cdn.organicmaps.app/subway/) and
+[this website](https://maps.vk.com/osm/tools/subways/latest/).
 You can find more info about this validator instance in
 [OSM Wiki](https://wiki.openstreetmap.org/wiki/Quality_assurance#subway-preprocessor).
 

scripts/process_subways.sh

@@ -6,13 +6,13 @@ if [ $# -gt 0 -a \( "${1-}" = "-h" -o "${1-}" = '--help' \) ]; then
 This script updates a planet or an extract, processes metro networks in it
 and produces a set of HTML files with validation results.
 
-Usage: $0 [<local/planet.osm.pbf | http://mirror.osm.ru/planet.osm.pbf>]
+Usage: $0 [<local/planet.{pbf,o5m} | http://mirror.osm.ru/planet.{pbf,o5m}>]
 
 In more detail, the script does the following:
   - If \$PLANET is a remote file, downloads it.
   - If \$BBOX variable is set, proceeds with this setting for the planet clipping. Otherwise uses \$POLY:
     unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro.
-  - Makes an extract of the \$PLANET using the *.poly file.
+  - Makes a *.o5m extract of the \$PLANET using the *.poly file.
   - Updates the extract.
   - Filters railway infrastructure from the extract.
   - Uses filtered file for validation and generates a bunch of output files.
@@ -28,8 +28,8 @@ variable is not defined or is null, otherwise they are kept.
 The \$PLANET file from remote URL is saved to a tempfile and is removed at the end.
 
 Environment variable reference:
-  - PLANET: path to a local or remote pbf source file (the entire planet or an extract)
-  - PLANET_METRO: path to a local pbf file with extract of cities having metro
+  - PLANET: path to a local or remote o5m or pbf source file (the entire planet or an extract)
+  - PLANET_METRO: path to a local o5m file with extract of cities having metro
     It's used instead of \$PLANET if exists otherwise it's created first
   - PLANET_UPDATE_SERVER: server to get replication data from. Defaults to https://planet.openstreetmap.org/replication/
   - CITIES_INFO_URL: http(s) or "file://" URL to a CSV file with reference information about rapid transit systems. A default value is hammered into python code.
@@ -48,6 +48,7 @@ Environment variable reference:
   - CITY_CACHE: json file with good cities obtained on previous validation runs
   - RECOVERY_PATH: file with some data collected at previous validation runs that
     may help to recover some simple validation errors
+  - OSMCTOOLS: path to osmconvert and osmupdate binaries
   - PYTHON: python 3 executable
   - GIT_PULL: set to 1 to update the scripts
   - TMPDIR: path to temporary files
@@ -66,7 +67,7 @@ function activate_venv_at_path() {
   path=$1
 
   if [ ! -d "$path/".venv ]; then
-    "${PYTHON:-python3}" -m venv "$path"/.venv
+    "${PYTHON:-python3.11}" -m venv "$path"/.venv
   fi
 
   source "$path"/.venv/bin/activate
@@ -78,10 +79,15 @@ function activate_venv_at_path() {
 }
 
 
-function check_osmium() {
-  if ! which osmium > /dev/null; then
-    echo "Please install osmium-tool"
-    exit 1
+function check_osmctools() {
+  OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}"
+  if [ ! -f "$OSMCTOOLS/osmupdate" ]; then
+    if which osmupdate > /dev/null; then
+      OSMCTOOLS="$(dirname "$(which osmupdate)")"
+    else
+      echo "Please compile osmctools to $OSMCTOOLS"
+      exit 1
+    fi
   fi
 }
 
@@ -157,7 +163,7 @@ if [ -n "${NEED_FILTER-}" ]; then
   fi
 
   if [ ! -f "${PLANET_METRO-}" ]; then
-    check_osmium
+    check_osmctools
     check_poly
 
     PLANET="${PLANET:-${1-}}"
@@ -180,7 +186,7 @@ if [ -n "${NEED_FILTER-}" ]; then
     fi
 
     if [ -z "${PLANET_METRO-}" ]; then
-      PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.osm.pbf")
+      PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.o5m")
       NEED_TO_REMOVE_PLANET_METRO=1
     fi
 
@@ -189,8 +195,10 @@ if [ -n "${NEED_FILTER-}" ]; then
       exit 6
     fi
 
-    osmium extract "$PLANET" \
-        ${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO"
+    mkdir -p $TMPDIR/osmconvert_temp/
+    "$OSMCTOOLS"/osmconvert "$PLANET" \
+        -t=$TMPDIR/osmconvert_temp/temp \
+        ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} -o="$PLANET_METRO"
   fi
 fi
 
@@ -202,38 +210,40 @@ fi
 
 # If there's no need to filter, then update is also unnecessary
 if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then
-  check_osmium
+  check_osmctools
   check_poly
-  PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/hour/}
+  PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/}
   PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")"
-  PLANET_METRO_ABS_NEW="$PLANET_METRO_ABS.new.osm.pbf"
   mkdir -p $TMPDIR/osmupdate_temp/
-
-  activate_venv_at_path "$SUBWAYS_REPO_PATH/scripts"
-  OSMUPDATE_ERRORS=$(pyosmium-up-to-date \
-                                 "$PLANET_METRO_ABS" \
-                                 --server $PLANET_UPDATE_SERVER \
-                                 --tmpdir $TMPDIR/osmupdate_temp/temp \
-                                 -o "$PLANET_METRO_ABS_NEW" 2>&1 || :)
-  deactivate
+  pushd $TMPDIR/osmupdate_temp/
+  export PATH="$PATH:$OSMCTOOLS"
+  OSMUPDATE_ERRORS=$(osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \
+                                 ${POLY:+"-B=$POLY"} "$PLANET_METRO_ABS" \
+                                 --base-url=$PLANET_UPDATE_SERVER \
+                                 --tempfiles=$TMPDIR/osmupdate_temp/temp \
+                                 "$PLANET_METRO_ABS.new.o5m" 2>&1 || :)
   if [ -n "$OSMUPDATE_ERRORS" ]; then
     echo "osmupdate failed: $OSMUPDATE_ERRORS"
     exit 7
   fi
-
-  # Since updating adds things outside the area, trim those again.
-  osmium extract "$PLANET_METRO_ABS_NEW" \
-      ${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO_ABS"
-  rm -f "$PLANET_METRO_ABS_NEW"
+  popd
+  mv "$PLANET_METRO_ABS.new.o5m" "$PLANET_METRO_ABS"
 fi
 
 # Filtering planet-metro
 
 if [ -n "${NEED_FILTER-}" ]; then
-  check_osmium
-  QRELATIONS="r/route,route_master=subway,light_rail,monorail,train r/public_transport=stop_area,stop_area_group"
-  QNODES="n/railway=station,subway_entrance,train_station_entrance n/station=subway,light_rail,monorail n/subway=yes n/light_rail=yes n/monorail=yes n/train=yes"
-  osmium tags-filter "$PLANET_METRO" $QRELATIONS $QNODES -o "$FILTERED_DATA" -O
+  check_osmctools
+  mkdir -p $TMPDIR/osmfilter_temp/
+  QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group"
+  QNODES="railway=station =subway_entrance =train_station_entrance station=subway =light_rail =monorail subway=yes light_rail=yes monorail=yes train=yes"
+  "$OSMCTOOLS/osmfilter" "$PLANET_METRO" \
+      --keep= \
+      --keep-relations="$QRELATIONS" \
+      --keep-nodes="$QNODES" \
+      --drop-author \
+      -t=$TMPDIR/osmfilter_temp/temp \
+      -o="$FILTERED_DATA"
 fi
 
 if [ -n "${NEED_TO_REMOVE_PLANET_METRO-}" ]; then
@@ -259,7 +269,6 @@ python "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ${QUIET:+-q} \
     -x "$FILTERED_DATA" -l "$VALIDATION" \
     ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
     ${MAPSME:+--output-mapsme "$MAPSME"} \
-    ${FMK:+--output-fmk "$FMK"} \
     ${GTFS:+--output-gtfs "$GTFS"} \
     ${CITY:+-c "$CITY"} \
     ${DUMP:+-d "$DUMP"} \

scripts/requirements.txt

@@ -1,2 +1 @@
-osmium
 -r ../subways/requirements.txt

subways/processors/__init__.py

@@ -1,8 +1,8 @@
 # Import only those processors (modules) you want to use.
 # Ignore F401 "module imported but unused" violation since these modules
 # are addressed via introspection.
-from . import gtfs, mapsme, fmk  # noqa F401
+from . import gtfs, mapsme  # noqa F401
 from ._common import transit_to_dict
 
 
-__all__ = ["gtfs", "mapsme", "fmk", "transit_to_dict"]
+__all__ = ["gtfs", "mapsme", "transit_to_dict"]

subways/processors/fmk.py

@@ -1,235 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import os
-import typing
-from collections import defaultdict
-from collections.abc import Callable
-from typing import Any, TypeAlias
-
-from subways.consts import DISPLACEMENT_TOLERANCE
-from subways.geom_utils import distance
-from subways.osm_element import el_center
-from subways.structure.station import Station
-from subways.types import IdT, LonLat, OsmElementT, TransfersT
-from ._common import (
-    DEFAULT_AVE_VEHICLE_SPEED,
-    DEFAULT_INTERVAL,
-    format_colour,
-    KMPH_TO_MPS,
-    SPEED_ON_TRANSFER,
-    TRANSFER_PENALTY,
-)
-
-if typing.TYPE_CHECKING:
-    from subways.structure.city import City
-    from subways.structure.stop_area import StopArea
-
-
-OSM_TYPES = {"n": (0, "node"), "w": (2, "way"), "r": (3, "relation")}
-ENTRANCE_PENALTY = 60  # seconds
-SPEED_TO_ENTRANCE = 5 * KMPH_TO_MPS  # m/s
-
-# (stoparea1_uid, stoparea2_uid) -> seconds; stoparea1_uid < stoparea2_uid
-TransferTimesT: TypeAlias = dict[tuple[int, int], int]
-
-
-def uid(elid: IdT, typ: str | None = None) -> int:
-    t = elid[0]
-    osm_id = int(elid[1:])
-    if not typ:
-        osm_id = (osm_id << 2) + OSM_TYPES[t][0]
-    elif typ != t:
-        raise Exception("Got {}, expected {}".format(elid, typ))
-    return osm_id << 1
-
-
-def transit_data_to_fmk(cities: list[City], transfers: TransfersT) -> dict:
-    """Generate all output and save to file.
-    :param cities: List of City instances
-    :param transfers: List of sets of StopArea.id
-    :param cache_path: Path to json-file with good cities cache or None.
-    """
-
-    def find_exits_for_platform(
-        center: LonLat, nodes: list[OsmElementT]
-    ) -> list[OsmElementT]:
-        exits: list[OsmElementT] = []
-        min_distance = None
-        for n in nodes:
-            d = distance(center, (n["lon"], n["lat"]))
-            if not min_distance:
-                min_distance = d * 2 / 3
-            elif d < min_distance:
-                continue
-            too_close = False
-            for e in exits:
-                d = distance((e["lon"], e["lat"]), (n["lon"], n["lat"]))
-                if d < min_distance:
-                    too_close = True
-                    break
-            if not too_close:
-                exits.append(n)
-        return exits
-
-    stop_areas: dict[IdT, StopArea] = {}
-    stops: dict[IdT, dict] = {}  # stoparea el_id -> stop jsonified data
-    networks = []
-    good_cities = [c for c in cities if c.is_good]
-    platform_nodes = {}
-
-    for city in good_cities:
-        network = {"network": city.name, "routes": [], "agency_id": city.id}
-        for route in city:
-            routes = {
-                "type": route.mode,
-                "ref": route.ref,
-                "name": route.name,
-                "colour": format_colour(route.colour),
-                "route_id": uid(route.id, "r"),
-                "itineraries": [],
-            }
-            if route.infill:
-                routes["casing"] = routes["colour"]
-                routes["colour"] = format_colour(route.infill)
-            for i, variant in enumerate(route):
-                itin = []
-                for stop in variant:
-                    stop_areas[stop.stoparea.id] = stop.stoparea
-                    itin.append(uid(stop.stoparea.id))
-                    # Make exits from platform nodes,
-                    # if we don't have proper exits
-                    if (
-                        len(stop.stoparea.entrances) + len(stop.stoparea.exits)
-                        == 0
-                    ):
-                        for pl in stop.stoparea.platforms:
-                            pl_el = city.elements[pl]
-                            if pl_el["type"] == "node":
-                                pl_nodes = [pl_el]
-                            elif pl_el["type"] == "way":
-                                pl_nodes = [
-                                    city.elements.get("n{}".format(n))
-                                    for n in pl_el["nodes"]
-                                ]
-                            else:
-                                pl_nodes = []
-                                for m in pl_el["members"]:
-                                    if m["type"] == "way":
-                                        if (
-                                            "{}{}".format(
-                                                m["type"][0], m["ref"]
-                                            )
-                                            in city.elements
-                                        ):
-                                            pl_nodes.extend(
-                                                [
-                                                    city.elements.get(
-                                                        "n{}".format(n)
-                                                    )
-                                                    for n in city.elements[
-                                                        "{}{}".format(
-                                                            m["type"][0],
-                                                            m["ref"],
-                                                        )
-                                                    ]["nodes"]
-                                                ]
-                                            )
-                            pl_nodes = [n for n in pl_nodes if n]
-                            platform_nodes[pl] = find_exits_for_platform(
-                                stop.stoparea.centers[pl], pl_nodes
-                            )
-
-                routes["itineraries"].append(itin)
-            network["routes"].append(routes)
-        networks.append(network)
-
-    for stop_id, stop in stop_areas.items():
-        st = {
-            "name": stop.name,
-            "int_name": stop.int_name,
-            "lat": stop.center[1],
-            "lon": stop.center[0],
-            "osm_type": OSM_TYPES[stop.station.id[0]][1],
-            "osm_id": int(stop.station.id[1:]),
-            "id": uid(stop.id),
-            "entrances": [],
-            "exits": [],
-        }
-        for e_l, k in ((stop.entrances, "entrances"), (stop.exits, "exits")):
-            for e in e_l:
-                if e[0] == "n":
-                    st[k].append(
-                        {
-                            "osm_type": "node",
-                            "osm_id": int(e[1:]),
-                            "lon": stop.centers[e][0],
-                            "lat": stop.centers[e][1],
-                        }
-                    )
-        if len(stop.entrances) + len(stop.exits) == 0:
-            if stop.platforms:
-                for pl in stop.platforms:
-                    for n in platform_nodes[pl]:
-                        for k in ("entrances", "exits"):
-                            st[k].append(
-                                {
-                                    "osm_type": n["type"],
-                                    "osm_id": n["id"],
-                                    "lon": n["lon"],
-                                    "lat": n["lat"],
-                                }
-                            )
-            else:
-                for k in ("entrances", "exits"):
-                    st[k].append(
-                        {
-                            "osm_type": OSM_TYPES[stop.station.id[0]][1],
-                            "osm_id": int(stop.station.id[1:]),
-                            "lon": stop.centers[stop.id][0],
-                            "lat": stop.centers[stop.id][1],
-                        }
-                    )
-
-        stops[stop_id] = st
-
-    pairwise_transfers: list[list[int]] = []
-    for stoparea_id_set in transfers:
-        tr = list(sorted([uid(sa_id) for sa_id in stoparea_id_set
-                          if sa_id in stops]))
-        if len(tr) > 1:
-            pairwise_transfers.append(tr)
-
-    result = {
-        "stops": list(stops.values()),
-        "transfers": pairwise_transfers,
-        "networks": networks,
-    }
-    return result
-
-
-def process(
-    cities: list[City],
-    transfers: TransfersT,
-    filename: str,
-    cache_path: str | None,
-) -> None:
-    """Generate all output and save to file.
-    :param cities: list of City instances
-    :param transfers: all collected transfers in the world
-    :param filename: Path to file to save the result
-    :param cache_path: Path to json-file with good cities cache or None.
-    """
-    if not filename.lower().endswith("json"):
-        filename = f"{filename}.json"
-
-    fmk_transit = transit_data_to_fmk(cities, transfers)
-
-    with open(filename, "w", encoding="utf-8") as f:
-        json.dump(
-            fmk_transit,
-            f,
-            indent=1,
-            ensure_ascii=False,
-        )

subways/requirements.txt

@@ -1 +1 @@
-lxml
+lxml==4.9.2