diff --git a/README.md b/README.md index e259087..b987e5f 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,24 @@ if you allow the `process_subway.py` to fetch data from Overpass API. Here are t python3 ./validation_to_html.py validation.log html ``` +## Publishing validation reports to the Web + +Expose a directory with static contents via a web-server and put into it: +- HTML files from the directory specified in the 2nd parameter of `validation_to_html.py` +- To vitalize "Y" (YAML), "J" (GeoJSON) and "M" (Map) links beside each city name: + - The contents of `render` directory from the repository + - `cities.txt` file generated with `--dump-city-list` parameter of `process_subways.py` + - YAML files created due to -d option of `process_subways.py` + - GeoJSON files created due to -j option of `process_subways.py` + + ## Related external resources Summary information about all metro networks that are monitored is gathered in the [Google Spreadsheet](https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k). Regular updates of validation results are available at -[this website](https://maps.mail.ru/osm/tools/subways/latest/). +[this website](https://maps.vk.com/osm/tools/subways/latest/). You can find more info about this validator instance in [OSM Wiki](https://wiki.openstreetmap.org/wiki/Quality_assurance#subway-preprocessor). diff --git a/mapsme_json_to_cities.py b/mapsme_json_to_cities.py index 1c69a77..736b74b 100644 --- a/mapsme_json_to_cities.py +++ b/mapsme_json_to_cities.py @@ -1,7 +1,15 @@ +""" +Generate sorted list of all cities, with [bad] mark for bad cities. + +!!! Deprecated for use in validation cycle. +Use "process_subways.py --dump-city-list " instead. +""" + + import argparse import json -from process_subways import DEFAULT_CITIES_INFO_URL, get_cities_info +from process_subways import BAD_MARK, DEFAULT_CITIES_INFO_URL, get_cities_info if __name__ == "__main__": @@ -56,7 +64,7 @@ if __name__ == "__main__": if ci["name"] in good_cities: lines.append(f"{ci['name']}, {ci['country']}") elif with_bad: - lines.append(f"{ci['name']}, {ci['country']} (Bad)") + lines.append(f"{ci['name']}, {ci['country']} {BAD_MARK}") for line in sorted(lines): print(line) diff --git a/process_subways.py b/process_subways.py index 1fd2262..ca71ed1 100755 --- a/process_subways.py +++ b/process_subways.py @@ -34,6 +34,7 @@ DEFAULT_CITIES_INFO_URL = ( "https://docs.google.com/spreadsheets/d/" f"{DEFAULT_SPREADSHEET_ID}/export?format=csv" ) +BAD_MARK = "[bad]" Point = tuple[float, float] @@ -69,7 +70,7 @@ def overpass_request( overground: bool, overpass_api: str, bboxes: list[list[float]] ) -> list[dict]: query = compose_overpass_request(overground, bboxes) - url = "{}?data={}".format(overpass_api, urllib.parse.quote(query)) + url = f"{overpass_api}?data={urllib.parse.quote(query)}" response = urllib.request.urlopen(url, timeout=1000) if (r_code := response.getcode()) != 200: raise Exception(f"Failed to query Overpass API: HTTP {r_code}") @@ -82,7 +83,7 @@ def multi_overpass( SLICE_SIZE = 10 INTERREQUEST_WAIT = 5 # in seconds result = [] - for i in range(0, len(bboxes) + SLICE_SIZE - 1, SLICE_SIZE): + for i in range(0, len(bboxes), SLICE_SIZE): if i > 0: time.sleep(INTERREQUEST_WAIT) bboxes_i = bboxes[i : i + SLICE_SIZE] # noqa E203 @@ -383,6 +384,14 @@ def main() -> None: type=argparse.FileType("w", encoding="utf-8"), help="Validation JSON file name", ) + parser.add_argument( + "--dump-city-list", + type=argparse.FileType("w", encoding="utf-8"), + help=( + "Dump sorted list of all city names, possibly with " + f"{BAD_MARK} mark" + ), + ) for processor_name, processor in inspect.getmembers( processors, inspect.ismodule @@ -496,6 +505,14 @@ def main() -> None: ", ".join(sorted(bad_city_names)), ) + if options.dump_city_list: + lines = sorted( + f"{city.name}, {city.country}" + f"{' ' + BAD_MARK if city.name in bad_city_names else ''}\n" + for city in cities + ) + options.dump_city_list.writelines(lines) + if options.recovery_path: write_recovery_data(options.recovery_path, recovery_data, cities) diff --git a/scripts/process_subways.sh b/scripts/process_subways.sh index 241b3c1..345dd2d 100755 --- a/scripts/process_subways.sh +++ b/scripts/process_subways.sh @@ -53,6 +53,7 @@ Environment variable reference: - GIT_PULL: set to 1 to update the scripts - TMPDIR: path to temporary files - HTML_DIR: target path for generated HTML files + - DUMP_CITY_LIST: file name to save sorted list of cities, with [bad] mark for bad cities - SERVER: server name and path to upload HTML files (e.g. ilya@osmz.ru:/var/www/) - SERVER_KEY: rsa key to supply for uploading the files - REMOVE_HTML: set to 1 to remove \$HTML_DIR after uploading @@ -246,7 +247,10 @@ VALIDATION="$TMPDIR/validation.json" ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \ ${MAPSME:+--output-mapsme "$MAPSME"} \ ${GTFS:+--output-gtfs "$GTFS"} \ - ${CITY:+-c "$CITY"} ${DUMP:+-d "$DUMP"} ${GEOJSON:+-j "$GEOJSON"} \ + ${CITY:+-c "$CITY"} \ + ${DUMP:+-d "$DUMP"} \ + ${GEOJSON:+-j "$GEOJSON"} \ + ${DUMP_CITY_LIST:+--dump-city-list "$DUMP_CITY_LIST"} \ ${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} \ ${CITY_CACHE:+--cache "$CITY_CACHE"} \ ${RECOVERY_PATH:+-r "$RECOVERY_PATH"} diff --git a/subway_structure.py b/subway_structure.py index e55bf71..7946e47 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -1024,7 +1024,7 @@ class Route: continue if Station.is_station(el, self.city.modes): - # A station may be not included into this route due to previous + # A station may be not included in this route due to previous # 'stop area has multiple stations' error. No other error # message is needed. pass @@ -2085,7 +2085,7 @@ class City: ): self.notice( f"Stop {st.stoparea.station.name} {st.stop} is included " - f"into the {route2.id} but not included into {route1.id}", + f"in the {route2.id} but not included in {route1.id}", route1.element, ) @@ -2103,7 +2103,7 @@ class City: ): self.notice( f"Stop {st.stoparea.station.name} {st.stop} is included " - f"into the {route1.id} but not included into {route2.id}", + f"in the {route1.id} but not included in {route2.id}", route2.element, ) diff --git a/tests/sample_data_for_error_messages.py b/tests/sample_data_for_error_messages.py index 245cfbb..2e20c73 100644 --- a/tests/sample_data_for_error_messages.py +++ b/tests/sample_data_for_error_messages.py @@ -342,7 +342,7 @@ metro_samples = [ 'Only one route in route_master. Please check if it needs a return route (relation 159, "C: 1-3-5-1")', # noqa: E501 'Route does not have a return direction (relation 163, "04: 1-2-3")', # noqa: E501 'Route does not have a return direction (relation 164, "04: 2-1")', # noqa: E501 - 'Stop Station 2 (1.0, 0.0) is included into the r203 but not included into r204 (relation 204, "2: 3-1")', # noqa: E501 + 'Stop Station 2 (1.0, 0.0) is included in the r203 but not included in r204 (relation 204, "2: 3-1")', # noqa: E501 'Route does not have a return direction (relation 205, "3: 1-2-3")', # noqa: E501 'Route does not have a return direction (relation 206, "3: 1-2-3")', # noqa: E501 'Route does not have a return direction (relation 207, "4: 4-3-2-1")', # noqa: E501 diff --git a/tests/sample_data_for_twin_routes.py b/tests/sample_data_for_twin_routes.py index 58b9e17..5847632 100644 --- a/tests/sample_data_for_twin_routes.py +++ b/tests/sample_data_for_twin_routes.py @@ -33,7 +33,7 @@ metro_samples = [ 'Route does not have a return direction (relation 157, "02: 4-1")', 'Route does not have a return direction (relation 158, "02: 1-3 (2)")', # noqa: E501 'Only one route in route_master. Please check if it needs a return route (relation 159, "C: 1-2-3-4-5-1")', # noqa: E501 - 'Stop Station 4 (3.0, 0.0) is included into the r205 but not included into r206 (relation 206, "3: 7-6-5-3-2-1")', # noqa: E501 + 'Stop Station 4 (3.0, 0.0) is included in the r205 but not included in r206 (relation 206, "3: 7-6-5-3-2-1")', # noqa: E501 'Route does not have a return direction (relation 207, "4: 4-3-2-1")', # noqa: E501 'Route does not have a return direction (relation 208, "4: 1-2-3-4")', # noqa: E501 'Route does not have a return direction (relation 209, "5: 1-2-3-5-6-7")', # noqa: E501 @@ -66,12 +66,12 @@ metro_samples = [ "notices": [ 'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 101, "1: 1-...-9-10-11-...-20")', # noqa: E501 'Should there be one stoparea or a transfer between Station 10 (0.09, 0.0) and Station 10(1) (0.09, 0.0003)? (relation 101, "1: 1-...-9-10-11-...-20")', # noqa: E501 - 'Stop Station 10 (0.09, 0.0) is included into the r105 but not included into r106 (relation 106, "3: 20-...-12-11(1)-9-...-1")', # noqa: E501 + 'Stop Station 10 (0.09, 0.0) is included in the r105 but not included in r106 (relation 106, "3: 20-...-12-11(1)-9-...-1")', # noqa: E501 'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 105, "3: 1-...-9-10-11-...-20")', # noqa: E501 - 'Stop Station 10 (0.09, 0.0) is included into the r107 but not included into r108 (relation 108, "4: 20-...12-11(2)-9-...-1")', # noqa: E501 + 'Stop Station 10 (0.09, 0.0) is included in the r107 but not included in r108 (relation 108, "4: 20-...12-11(2)-9-...-1")', # noqa: E501 'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 201, "11: 1-...-9-10-11-...-20")', # noqa: E501 'Should there be one stoparea or a transfer between Station 10 (0.09, 0.0) and Station 10(1) (0.09, 0.0003)? (relation 201, "11: 1-...-9-10-11-...-20")', # noqa: E501 - 'Stop Station 10 (0.09, 0.0) is included into the r205 but not included into r206 (relation 206, "13: 20-...-12-11(1)-9-...-1")', # noqa: E501 + 'Stop Station 10 (0.09, 0.0) is included in the r205 but not included in r206 (relation 206, "13: 20-...-12-11(1)-9-...-1")', # noqa: E501 'Should there be one stoparea or a transfer between Station 11 (0.1, 0.0) and Station 11(1) (0.1, 0.0003)? (relation 205, "13: 1-...-9-10-11-...-20")', # noqa: E501 ], },