Merge remote-tracking branch 'zverik/master'

.gitignore

@@ -13,3 +13,5 @@ html/
 *.pyc
 *.txt
 *.zip
+*.osm.pbf
+*.o5m

README.md

@@ -34,8 +34,7 @@ for details. Here is an example of the script usage:
 
 ```bash
 export PLANET=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf
-export PLANET_METRO="$HOME/metro/planet-metro.o5m
-export OSMCTOOLS="$HOME/osmctools"
+export PLANET_METRO="$HOME/metro/planet-metro.osm.pbf
 export TMPDIR="$HOME/metro/tmp"
 export HTML_DIR="$HOME/metro/tmp_html"
 export DUMP="$HTML_DIR"
@@ -60,14 +59,15 @@ if you allow the `scripts/process_subway.py` to fetch data from Overpass API. He
     cd subways_validator
    ```
 3. Configure python environment, e.g.
-   ```bash
-   python3 -m venv scripts/.venv
-   source scripts/.venv/bin/activate
-   pip install scripts/requirements.txt
-   ```
+    ```bash
+    python3 -m venv scripts/.venv
+    source scripts/.venv/bin/activate
+    pip install -r scripts/requirements.txt
+    ```
+    (this is optional if you only process a single city though.)
 4. Execute
     ```bash
-    python3 scripts/process_subways.py -c "London" \
+    PYTHONPATH=. python3 scripts/process_subways.py -c "London" \
         -l validation.log -d London.yaml
     ```
     here
@@ -101,8 +101,8 @@ Expose a directory with static contents via a web-server and put into it:
 Summary information about all metro networks that are monitored is gathered in the
 [Google Spreadsheet](https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k).
 
-Regular updates of validation results are available at [Organic Maps](https://cdn.organicmaps.app/subway/) and
-[this website](https://maps.vk.com/osm/tools/subways/latest/).
+Regular updates of validation results are available at
+[this website](https://cdn.organicmaps.app/subway).
 You can find more info about this validator instance in
 [OSM Wiki](https://wiki.openstreetmap.org/wiki/Quality_assurance#subway-preprocessor).
 

scripts/process_subways.sh

@@ -6,13 +6,13 @@ if [ $# -gt 0 -a \( "${1-}" = "-h" -o "${1-}" = '--help' \) ]; then
 This script updates a planet or an extract, processes metro networks in it
 and produces a set of HTML files with validation results.
 
-Usage: $0 [<local/planet.{pbf,o5m} | http://mirror.osm.ru/planet.{pbf,o5m}>]
+Usage: $0 [<local/planet.osm.pbf | http://mirror.osm.ru/planet.osm.pbf>]
 
 In more detail, the script does the following:
   - If \$PLANET is a remote file, downloads it.
   - If \$BBOX variable is set, proceeds with this setting for the planet clipping. Otherwise uses \$POLY:
     unless \$POLY variable is set and the file exists, generates a *.poly file with union of bboxes of all cities having metro.
-  - Makes a *.o5m extract of the \$PLANET using the *.poly file.
+  - Makes an extract of the \$PLANET using the *.poly file.
   - Updates the extract.
   - Filters railway infrastructure from the extract.
   - Uses filtered file for validation and generates a bunch of output files.
@@ -28,8 +28,8 @@ variable is not defined or is null, otherwise they are kept.
 The \$PLANET file from remote URL is saved to a tempfile and is removed at the end.
 
 Environment variable reference:
-  - PLANET: path to a local or remote o5m or pbf source file (the entire planet or an extract)
-  - PLANET_METRO: path to a local o5m file with extract of cities having metro
+  - PLANET: path to a local or remote pbf source file (the entire planet or an extract)
+  - PLANET_METRO: path to a local pbf file with extract of cities having metro
     It's used instead of \$PLANET if exists otherwise it's created first
   - PLANET_UPDATE_SERVER: server to get replication data from. Defaults to https://planet.openstreetmap.org/replication/
   - CITIES_INFO_URL: http(s) or "file://" URL to a CSV file with reference information about rapid transit systems. A default value is hammered into python code.
@@ -48,7 +48,6 @@ Environment variable reference:
   - CITY_CACHE: json file with good cities obtained on previous validation runs
   - RECOVERY_PATH: file with some data collected at previous validation runs that
     may help to recover some simple validation errors
-  - OSMCTOOLS: path to osmconvert and osmupdate binaries
   - PYTHON: python 3 executable
   - GIT_PULL: set to 1 to update the scripts
   - TMPDIR: path to temporary files
@@ -67,7 +66,7 @@ function activate_venv_at_path() {
   path=$1
 
   if [ ! -d "$path/".venv ]; then
-    "${PYTHON:-python3.11}" -m venv "$path"/.venv
+    "${PYTHON:-python3}" -m venv "$path"/.venv
   fi
 
   source "$path"/.venv/bin/activate
@@ -79,15 +78,10 @@ function activate_venv_at_path() {
 }
 
 
-function check_osmctools() {
-  OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}"
-  if [ ! -f "$OSMCTOOLS/osmupdate" ]; then
-    if which osmupdate > /dev/null; then
-      OSMCTOOLS="$(dirname "$(which osmupdate)")"
-    else
-      echo "Please compile osmctools to $OSMCTOOLS"
-      exit 1
-    fi
+function check_osmium() {
+  if ! which osmium > /dev/null; then
+    echo "Please install osmium-tool"
+    exit 1
   fi
 }
 
@@ -163,7 +157,7 @@ if [ -n "${NEED_FILTER-}" ]; then
   fi
 
   if [ ! -f "${PLANET_METRO-}" ]; then
-    check_osmctools
+    check_osmium
     check_poly
 
     PLANET="${PLANET:-${1-}}"
@@ -186,7 +180,7 @@ if [ -n "${NEED_FILTER-}" ]; then
     fi
 
     if [ -z "${PLANET_METRO-}" ]; then
-      PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.o5m")
+      PLANET_METRO=$(mktemp "$TMPDIR/planet-metro.XXXXXXXX.osm.pbf")
       NEED_TO_REMOVE_PLANET_METRO=1
     fi
 
@@ -195,10 +189,8 @@ if [ -n "${NEED_FILTER-}" ]; then
       exit 6
     fi
 
-    mkdir -p $TMPDIR/osmconvert_temp/
-    "$OSMCTOOLS"/osmconvert "$PLANET" \
-        -t=$TMPDIR/osmconvert_temp/temp \
-        ${BBOX:+"-b=$BBOX"} ${POLY:+"-B=$POLY"} -o="$PLANET_METRO"
+    osmium extract "$PLANET" \
+        ${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO"
   fi
 fi
 
@@ -210,40 +202,38 @@ fi
 
 # If there's no need to filter, then update is also unnecessary
 if [ -z "${SKIP_PLANET_UPDATE-}" -a -n "${NEED_FILTER-}" ]; then
-  check_osmctools
+  check_osmium
   check_poly
-  PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/}
+  PLANET_UPDATE_SERVER=${PLANET_UPDATE_SERVER:-https://planet.openstreetmap.org/replication/hour/}
   PLANET_METRO_ABS="$(cd "$(dirname "$PLANET_METRO")"; pwd)/$(basename "$PLANET_METRO")"
+  PLANET_METRO_ABS_NEW="$PLANET_METRO_ABS.new.osm.pbf"
   mkdir -p $TMPDIR/osmupdate_temp/
-  pushd $TMPDIR/osmupdate_temp/
-  export PATH="$PATH:$OSMCTOOLS"
-  OSMUPDATE_ERRORS=$(osmupdate --drop-author --out-o5m ${BBOX:+"-b=$BBOX"} \
-                                 ${POLY:+"-B=$POLY"} "$PLANET_METRO_ABS" \
-                                 --base-url=$PLANET_UPDATE_SERVER \
-                                 --tempfiles=$TMPDIR/osmupdate_temp/temp \
-                                 "$PLANET_METRO_ABS.new.o5m" 2>&1 || :)
+
+  activate_venv_at_path "$SUBWAYS_REPO_PATH/scripts"
+  OSMUPDATE_ERRORS=$(pyosmium-up-to-date \
+                                 "$PLANET_METRO_ABS" \
+                                 --server $PLANET_UPDATE_SERVER \
+                                 --tmpdir $TMPDIR/osmupdate_temp/temp \
+                                 -o "$PLANET_METRO_ABS_NEW" 2>&1 || :)
+  deactivate
   if [ -n "$OSMUPDATE_ERRORS" ]; then
     echo "osmupdate failed: $OSMUPDATE_ERRORS"
     exit 7
   fi
-  popd
-  mv "$PLANET_METRO_ABS.new.o5m" "$PLANET_METRO_ABS"
+
+  # Since updating adds things outside the area, trim those again.
+  osmium extract "$PLANET_METRO_ABS_NEW" \
+      ${BBOX:+"--bbox=$BBOX"} ${POLY:+"--polygon=$POLY"} -O -o "$PLANET_METRO_ABS"
+  rm -f "$PLANET_METRO_ABS_NEW"
 fi
 
 # Filtering planet-metro
 
 if [ -n "${NEED_FILTER-}" ]; then
-  check_osmctools
-  mkdir -p $TMPDIR/osmfilter_temp/
-  QRELATIONS="route=subway =light_rail =monorail =train route_master=subway =light_rail =monorail =train public_transport=stop_area =stop_area_group"
-  QNODES="railway=station =subway_entrance =train_station_entrance station=subway =light_rail =monorail subway=yes light_rail=yes monorail=yes train=yes"
-  "$OSMCTOOLS/osmfilter" "$PLANET_METRO" \
-      --keep= \
-      --keep-relations="$QRELATIONS" \
-      --keep-nodes="$QNODES" \
-      --drop-author \
-      -t=$TMPDIR/osmfilter_temp/temp \
-      -o="$FILTERED_DATA"
+  check_osmium
+  QRELATIONS="r/route,route_master=subway,light_rail,monorail,train r/public_transport=stop_area,stop_area_group"
+  QNODES="n/railway=station,subway_entrance,train_station_entrance n/station=subway,light_rail,monorail n/subway=yes n/light_rail=yes n/monorail=yes n/train=yes"
+  osmium tags-filter "$PLANET_METRO" $QRELATIONS $QNODES -o "$FILTERED_DATA" -O
 fi
 
 if [ -n "${NEED_TO_REMOVE_PLANET_METRO-}" ]; then
@@ -269,6 +259,7 @@ python "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ${QUIET:+-q} \
     -x "$FILTERED_DATA" -l "$VALIDATION" \
     ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
     ${MAPSME:+--output-mapsme "$MAPSME"} \
+    ${FMK:+--output-fmk "$FMK"} \
     ${GTFS:+--output-gtfs "$GTFS"} \
     ${CITY:+-c "$CITY"} \
     ${DUMP:+-d "$DUMP"} \

scripts/requirements.txt

@@ -1 +1,2 @@
+osmium
 -r ../subways/requirements.txt

subways/processors/__init__.py

@@ -1,8 +1,8 @@
 # Import only those processors (modules) you want to use.
 # Ignore F401 "module imported but unused" violation since these modules
 # are addressed via introspection.
-from . import gtfs, mapsme  # noqa F401
+from . import gtfs, mapsme, fmk  # noqa F401
 from ._common import transit_to_dict
 
 
-__all__ = ["gtfs", "mapsme", "transit_to_dict"]
+__all__ = ["gtfs", "mapsme", "fmk", "transit_to_dict"]

subways/processors/fmk.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import typing
+from collections import defaultdict
+from collections.abc import Callable
+from typing import Any, TypeAlias
+
+from subways.consts import DISPLACEMENT_TOLERANCE
+from subways.geom_utils import distance
+from subways.osm_element import el_center
+from subways.structure.station import Station
+from subways.types import IdT, LonLat, OsmElementT, TransfersT
+from ._common import (
+    DEFAULT_AVE_VEHICLE_SPEED,
+    DEFAULT_INTERVAL,
+    format_colour,
+    KMPH_TO_MPS,
+    SPEED_ON_TRANSFER,
+    TRANSFER_PENALTY,
+)
+
+if typing.TYPE_CHECKING:
+    from subways.structure.city import City
+    from subways.structure.stop_area import StopArea
+
+
+OSM_TYPES = {"n": (0, "node"), "w": (2, "way"), "r": (3, "relation")}
+ENTRANCE_PENALTY = 60  # seconds
+SPEED_TO_ENTRANCE = 5 * KMPH_TO_MPS  # m/s
+
+# (stoparea1_uid, stoparea2_uid) -> seconds; stoparea1_uid < stoparea2_uid
+TransferTimesT: TypeAlias = dict[tuple[int, int], int]
+
+
+def uid(elid: IdT, typ: str | None = None) -> int:
+    t = elid[0]
+    osm_id = int(elid[1:])
+    if not typ:
+        osm_id = (osm_id << 2) + OSM_TYPES[t][0]
+    elif typ != t:
+        raise Exception("Got {}, expected {}".format(elid, typ))
+    return osm_id << 1
+
+
+def transit_data_to_fmk(cities: list[City], transfers: TransfersT) -> dict:
+    """Generate all output and save to file.
+    :param cities: List of City instances
+    :param transfers: List of sets of StopArea.id
+    :param cache_path: Path to json-file with good cities cache or None.
+    """
+
+    def find_exits_for_platform(
+        center: LonLat, nodes: list[OsmElementT]
+    ) -> list[OsmElementT]:
+        exits: list[OsmElementT] = []
+        min_distance = None
+        for n in nodes:
+            d = distance(center, (n["lon"], n["lat"]))
+            if not min_distance:
+                min_distance = d * 2 / 3
+            elif d < min_distance:
+                continue
+            too_close = False
+            for e in exits:
+                d = distance((e["lon"], e["lat"]), (n["lon"], n["lat"]))
+                if d < min_distance:
+                    too_close = True
+                    break
+            if not too_close:
+                exits.append(n)
+        return exits
+
+    stop_areas: dict[IdT, StopArea] = {}
+    stops: dict[IdT, dict] = {}  # stoparea el_id -> stop jsonified data
+    networks = []
+    good_cities = [c for c in cities if c.is_good]
+    platform_nodes = {}
+
+    for city in good_cities:
+        network = {"network": city.name, "routes": [], "agency_id": city.id}
+        for route in city:
+            routes = {
+                "type": route.mode,
+                "ref": route.ref,
+                "name": route.name,
+                "colour": format_colour(route.colour),
+                "route_id": uid(route.id, "r"),
+                "itineraries": [],
+            }
+            if route.infill:
+                routes["casing"] = routes["colour"]
+                routes["colour"] = format_colour(route.infill)
+            for i, variant in enumerate(route):
+                itin = []
+                for stop in variant:
+                    stop_areas[stop.stoparea.id] = stop.stoparea
+                    itin.append(uid(stop.stoparea.id))
+                    # Make exits from platform nodes,
+                    # if we don't have proper exits
+                    if (
+                        len(stop.stoparea.entrances) + len(stop.stoparea.exits)
+                        == 0
+                    ):
+                        for pl in stop.stoparea.platforms:
+                            pl_el = city.elements[pl]
+                            if pl_el["type"] == "node":
+                                pl_nodes = [pl_el]
+                            elif pl_el["type"] == "way":
+                                pl_nodes = [
+                                    city.elements.get("n{}".format(n))
+                                    for n in pl_el["nodes"]
+                                ]
+                            else:
+                                pl_nodes = []
+                                for m in pl_el["members"]:
+                                    if m["type"] == "way":
+                                        if (
+                                            "{}{}".format(
+                                                m["type"][0], m["ref"]
+                                            )
+                                            in city.elements
+                                        ):
+                                            pl_nodes.extend(
+                                                [
+                                                    city.elements.get(
+                                                        "n{}".format(n)
+                                                    )
+                                                    for n in city.elements[
+                                                        "{}{}".format(
+                                                            m["type"][0],
+                                                            m["ref"],
+                                                        )
+                                                    ]["nodes"]
+                                                ]
+                                            )
+                            pl_nodes = [n for n in pl_nodes if n]
+                            platform_nodes[pl] = find_exits_for_platform(
+                                stop.stoparea.centers[pl], pl_nodes
+                            )
+
+                routes["itineraries"].append(itin)
+            network["routes"].append(routes)
+        networks.append(network)
+
+    for stop_id, stop in stop_areas.items():
+        st = {
+            "name": stop.name,
+            "int_name": stop.int_name,
+            "lat": stop.center[1],
+            "lon": stop.center[0],
+            "osm_type": OSM_TYPES[stop.station.id[0]][1],
+            "osm_id": int(stop.station.id[1:]),
+            "id": uid(stop.id),
+            "entrances": [],
+            "exits": [],
+        }
+        for e_l, k in ((stop.entrances, "entrances"), (stop.exits, "exits")):
+            for e in e_l:
+                if e[0] == "n":
+                    st[k].append(
+                        {
+                            "osm_type": "node",
+                            "osm_id": int(e[1:]),
+                            "lon": stop.centers[e][0],
+                            "lat": stop.centers[e][1],
+                        }
+                    )
+        if len(stop.entrances) + len(stop.exits) == 0:
+            if stop.platforms:
+                for pl in stop.platforms:
+                    for n in platform_nodes[pl]:
+                        for k in ("entrances", "exits"):
+                            st[k].append(
+                                {
+                                    "osm_type": n["type"],
+                                    "osm_id": n["id"],
+                                    "lon": n["lon"],
+                                    "lat": n["lat"],
+                                }
+                            )
+            else:
+                for k in ("entrances", "exits"):
+                    st[k].append(
+                        {
+                            "osm_type": OSM_TYPES[stop.station.id[0]][1],
+                            "osm_id": int(stop.station.id[1:]),
+                            "lon": stop.centers[stop.id][0],
+                            "lat": stop.centers[stop.id][1],
+                        }
+                    )
+
+        stops[stop_id] = st
+
+    pairwise_transfers: list[list[int]] = []
+    for stoparea_id_set in transfers:
+        tr = list(sorted([uid(sa_id) for sa_id in stoparea_id_set
+                          if sa_id in stops]))
+        if len(tr) > 1:
+            pairwise_transfers.append(tr)
+
+    result = {
+        "stops": list(stops.values()),
+        "transfers": pairwise_transfers,
+        "networks": networks,
+    }
+    return result
+
+
+def process(
+    cities: list[City],
+    transfers: TransfersT,
+    filename: str,
+    cache_path: str | None,
+) -> None:
+    """Generate all output and save to file.
+    :param cities: list of City instances
+    :param transfers: all collected transfers in the world
+    :param filename: Path to file to save the result
+    :param cache_path: Path to json-file with good cities cache or None.
+    """
+    if not filename.lower().endswith("json"):
+        filename = f"{filename}.json"
+
+    fmk_transit = transit_data_to_fmk(cities, transfers)
+
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(
+            fmk_transit,
+            f,
+            indent=1,
+            ensure_ascii=False,
+        )

subways/requirements.txt

@@ -1 +1 @@
-lxml==4.9.2
+lxml