organicmaps/subways
13
Fork 0
Code Issues Pull requests 1 Projects Releases Wiki Activity Actions

Add --dump-city-list option to untie cities.txt formation from mapsme.json

Browse source
This commit is contained in:
Alexey Zakharenkov 2023-12-19 13:44:04 +03:00 • committed by Alexey Zakharenkov
parent 970b4a51ee
commit 1e4e434d49
7 changed files with 54 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
README.md
View file

@ -79,13 +79,24 @@ if you allow the `process_subway.py` to fetch data from Overpass API. Here are t
python3 ./validation_to_html.py validation.log html
```
## Publishing validation reports to the Web
Expose a directory with static contents via a web-server and put into it:
- HTML files from the directory specified in the 2nd parameter of `validation_to_html.py`
- To vitalize "Y" (YAML), "J" (GeoJSON) and "M" (Map) links beside each city name:
- The contents of `render` directory from the repository
- `cities.txt` file generated with `--dump-city-list` parameter of `process_subways.py`
- YAML files created due to -d option of `process_subways.py`
- GeoJSON files created due to -j option of `process_subways.py`
## Related external resources
Summary information about all metro networks that are monitored is gathered in the
[Google Spreadsheet](https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k).
Regular updates of validation results are available at
[this website](https://maps.mail.ru/osm/tools/subways/latest/).
[this website](https://maps.vk.com/osm/tools/subways/latest/).
You can find more info about this validator instance in
[OSM Wiki](https://wiki.openstreetmap.org/wiki/Quality_assurance#subway-preprocessor).

12
mapsme_json_to_cities.py
View file

@ -1,7 +1,15 @@
"""
Generate sorted list of all cities, with [bad] mark for bad cities.
!!! Deprecated for use in validation cycle.
Use "process_subways.py --dump-city-list <filename>" instead.
"""
import argparse
import json
from process_subways import DEFAULT_CITIES_INFO_URL, get_cities_info
from process_subways import BAD_MARK, DEFAULT_CITIES_INFO_URL, get_cities_info
if __name__ == "__main__":
@ -56,7 +64,7 @@ if __name__ == "__main__":
if ci["name"] in good_cities:
lines.append(f"{ci['name']}, {ci['country']}")
elif with_bad:
lines.append(f"{ci['name']}, {ci['country']} (Bad)")
lines.append(f"{ci['name']}, {ci['country']} {BAD_MARK}")
for line in sorted(lines):
print(line)

21
process_subways.py
View file

@ -34,6 +34,7 @@ DEFAULT_CITIES_INFO_URL = (
"https://docs.google.com/spreadsheets/d/"
f"{DEFAULT_SPREADSHEET_ID}/export?format=csv"
)
BAD_MARK = "[bad]"
Point = tuple[float, float]
@ -69,7 +70,7 @@ def overpass_request(
overground: bool, overpass_api: str, bboxes: list[list[float]]
) -> list[dict]:
query = compose_overpass_request(overground, bboxes)
url = "{}?data={}".format(overpass_api, urllib.parse.quote(query))
url = f"{overpass_api}?data={urllib.parse.quote(query)}"
response = urllib.request.urlopen(url, timeout=1000)
if (r_code := response.getcode()) != 200:
raise Exception(f"Failed to query Overpass API: HTTP {r_code}")
@ -82,7 +83,7 @@ def multi_overpass(
SLICE_SIZE = 10
INTERREQUEST_WAIT = 5 # in seconds
result = []
for i in range(0, len(bboxes) + SLICE_SIZE - 1, SLICE_SIZE):
for i in range(0, len(bboxes), SLICE_SIZE):
if i > 0:
time.sleep(INTERREQUEST_WAIT)
bboxes_i = bboxes[i : i + SLICE_SIZE] # noqa E203
@ -383,6 +384,14 @@ def main() -> None:
type=argparse.FileType("w", encoding="utf-8"),
help="Validation JSON file name",
)
parser.add_argument(
"--dump-city-list",
type=argparse.FileType("w", encoding="utf-8"),
help=(
"Dump sorted list of all city names, possibly with "
f"{BAD_MARK} mark"
),
)
for processor_name, processor in inspect.getmembers(
processors, inspect.ismodule
@ -496,6 +505,14 @@ def main() -> None:
", ".join(sorted(bad_city_names)),
)
if options.dump_city_list:
lines = sorted(
f"{city.name}, {city.country}"
f"{' ' + BAD_MARK if city.name in bad_city_names else ''}\n"
for city in cities
)
options.dump_city_list.writelines(lines)
if options.recovery_path:
write_recovery_data(options.recovery_path, recovery_data, cities)

6
scripts/process_subways.sh
View file

@ -53,6 +53,7 @@ Environment variable reference:
- GIT_PULL: set to 1 to update the scripts
- TMPDIR: path to temporary files
- HTML_DIR: target path for generated HTML files
- DUMP_CITY_LIST: file name to save sorted list of cities, with [bad] mark for bad cities
- SERVER: server name and path to upload HTML files (e.g. ilya@osmz.ru:/var/www/)
- SERVER_KEY: rsa key to supply for uploading the files
- REMOVE_HTML: set to 1 to remove \$HTML_DIR after uploading
@ -246,7 +247,10 @@ VALIDATION="$TMPDIR/validation.json"
${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
${MAPSME:+--output-mapsme "$MAPSME"} \
${GTFS:+--output-gtfs "$GTFS"} \
${CITY:+-c "$CITY"} ${DUMP:+-d "$DUMP"} ${GEOJSON:+-j "$GEOJSON"} \
${CITY:+-c "$CITY"} \
${DUMP:+-d "$DUMP"} \
${GEOJSON:+-j "$GEOJSON"} \
${DUMP_CITY_LIST:+--dump-city-list "$DUMP_CITY_LIST"} \
${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} \
${CITY_CACHE:+--cache "$CITY_CACHE"} \
${RECOVERY_PATH:+-r "$RECOVERY_PATH"}

6
subway_structure.py
View file

@ -1024,7 +1024,7 @@ class Route:
continue
if Station.is_station(el, self.city.modes):
# A station may be not included into this route due to previous
# A station may be not included in this route due to previous
# 'stop area has multiple stations' error. No other error
# message is needed.
pass
@ -2085,7 +2085,7 @@ class City:
):
self.notice(
f"Stop {st.stoparea.station.name} {st.stop} is included "
f"into the {route2.id} but not included into {route1.id}",
f"in the {route2.id} but not included in {route1.id}",
route1.element,
)
@ -2103,7 +2103,7 @@ class City:
):
self.notice(
f"Stop {st.stoparea.station.name} {st.stop} is included "
f"into the {route1.id} but not included into {route2.id}",
f"in the {route1.id} but not included in {route2.id}",
route2.element,
)

2
tests/sample_data_for_error_messages.py
View file

@ -342,7 +342,7 @@ metro_samples = [
'Only one route in route_master. Please check if it needs a return route (relation 159, "C: 1-3-5-1")', # noqa: E501
'Route does not have a return direction (relation 163, "04: 1-2-3")', # noqa: E501
'Route does not have a return direction (relation 164, "04: 2-1")', # noqa: E501
'Stop Station 2 (1.0, 0.0) is included into the r203 but not included into r204 (relation 204, "2: 3-1")', # noqa: E501
'Stop Station 2 (1.0, 0.0) is included in the r203 but not included in r204 (relation 204, "2: 3-1")', # noqa: E501
'Route does not have a return direction (relation 205, "3: 1-2-3")', # noqa: E501
'Route does not have a return direction (relation 206, "3: 1-2-3")', # noqa: E501
'Route does not have a return direction (relation 207, "4: 4-3-2-1")', # noqa: E501

8
tests/sample_data_for_twin_routes.py
View file

@ -33,7 +33,7 @@ metro_samples = [
'Route does not have a return direction (relation 157, "02: 4-1")',
'Route does not have a return direction (relation 158, "02: 1-3 (2)")', # noqa: E501
'Only one route in route_master. Please check if it needs a return route (relation 159, "C: 1-2-3-4-5-1")', # noqa: E501
'Stop Station 4 (3.0, 0.0) is included into the r205 but not included into r206 (relation 206, "3: 7-6-5-3-2-1")', # noqa: E501
'Stop Station 4 (3.0, 0.0) is included in the r205 but not included in r206 (relation 206, "3: 7-6-5-3-2-1")', # noqa: E501
'Route does not have a return direction (relation 207, "4: 4-3-2-1")', # noqa: E501
'Route does not have a return direction (relation 208, "4: 1-2-3-4")', # noqa: E501
'Route does not have a return direction (relation 209, "5: 1-2-3-5-6-7")', # noqa: E501
@ -66,12 +66,12 @@ metro_samples = [
"notices": [
'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 101, "1: 1-...-9-10-11-...-20")', # noqa: E501
'Should there be one stoparea or a transfer between Station 10 (0.09, 0.0) and Station 10(1) (0.09, 0.0003)? (relation 101, "1: 1-...-9-10-11-...-20")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included into the r105 but not included into r106 (relation 106, "3: 20-...-12-11(1)-9-...-1")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included in the r105 but not included in r106 (relation 106, "3: 20-...-12-11(1)-9-...-1")', # noqa: E501
'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 105, "3: 1-...-9-10-11-...-20")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included into the r107 but not included into r108 (relation 108, "4: 20-...12-11(2)-9-...-1")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included in the r107 but not included in r108 (relation 108, "4: 20-...12-11(2)-9-...-1")', # noqa: E501
'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 201, "11: 1-...-9-10-11-...-20")', # noqa: E501
'Should there be one stoparea or a transfer between Station 10 (0.09, 0.0) and Station 10(1) (0.09, 0.0003)? (relation 201, "11: 1-...-9-10-11-...-20")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included into the r205 but not included into r206 (relation 206, "13: 20-...-12-11(1)-9-...-1")', # noqa: E501
'Stop Station 10 (0.09, 0.0) is included in the r205 but not included in r206 (relation 206, "13: 20-...-12-11(1)-9-...-1")', # noqa: E501
'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 205, "13: 1-...-9-10-11-...-20")', # noqa: E501
],
},