diff --git a/process_subways.py b/process_subways.py index 2dce57b..1e0d8dc 100755 --- a/process_subways.py +++ b/process_subways.py @@ -291,6 +291,8 @@ if __name__ == '__main__': help='Use city boundaries to query Overpass API instead of querying the world') parser.add_argument('-q', '--quiet', action='store_true', help='Show only warnings and errors') parser.add_argument('-c', '--city', help='Validate only a single city or a country') + parser.add_argument('-t', '--overground', action='store_true', + help='Process overground transport instead of subways') parser.add_argument('-e', '--entrances', type=argparse.FileType('w', encoding='utf-8'), help='Export unused subway entrances as GeoJSON here') parser.add_argument('-l', '--log', type=argparse.FileType('w', encoding='utf-8'), @@ -312,7 +314,7 @@ if __name__ == '__main__': format='%(asctime)s %(levelname)-7s %(message)s') # Downloading cities from Google Spreadsheets - cities = download_cities() + cities = download_cities(options.overground) if options.city: cities = [c for c in cities if c.name == options.city or c.country == options.city] if not cities: diff --git a/subway_structure.py b/subway_structure.py index df54f4e..0736a5f 100644 --- a/subway_structure.py +++ b/subway_structure.py @@ -15,6 +15,7 @@ ALLOWED_TRANSFERS_MISMATCH = 0.07 # part of total interchanges count MIN_ANGLE_BETWEEN_STOPS = 45 # in degrees DEFAULT_MODES = set(('subway', 'light_rail')) +DEFAULT_MODES_OVERGROUND = set(('tram',)) # TODO: bus and trolleybus? ALL_MODES = set(('subway', 'light_rail', 'monorail', 'train', 'tram', 'bus', 'trolleybus', 'aerialway', 'ferry')) RAILWAY_TYPES = set(('rail', 'light_rail', 'subway', 'narrow_gauge', @@ -186,6 +187,8 @@ class Station: def is_station(el, modes=DEFAULT_MODES): # public_transport=station is too ambigous and unspecific to use, # so we expect for it to be backed by railway=station. + if 'tram' in modes and el.get('tags', {}).get('railway') == 'tram_stop': + return True if el.get('tags', {}).get('railway') not in ('station', 'halt'): return False for k in CONSTRUCTION_KEYS: @@ -605,7 +608,8 @@ class Route: self.ref = relation['tags'].get('ref', master_tags.get( 'ref', relation['tags'].get('name', None))) self.name = relation['tags'].get('name', None) - if 'colour' not in relation['tags'] and 'colour' not in master_tags: + self.mode = relation['tags']['route'] + if 'colour' not in relation['tags'] and 'colour' not in master_tags and self.mode != 'tram': city.warn('Missing colour on a route', relation) try: self.colour = normalize_colour(relation['tags'].get( @@ -620,7 +624,6 @@ class Route: self.infill = None city.warn(str(e), relation) self.network = Route.get_network(relation) - self.mode = relation['tags']['route'] self.interval = Route.get_interval(relation['tags']) or Route.get_interval(master_tags) if relation['tags'].get('public_transport:version') == '1': city.error('Public transport version is 1, which means the route ' @@ -866,17 +869,24 @@ class RouteMaster: class City: - def __init__(self, row): + def __init__(self, row, overground=False): self.name = row[1] self.country = row[2] self.continent = row[3] if not row[0]: self.error('City {} does not have an id'.format(self.name)) self.id = int(row[0] or '0') - self.num_stations = int(row[4]) - self.num_lines = int(row[5] or '0') - self.num_light_lines = int(row[6] or '0') - self.num_interchanges = int(row[7] or '0') + self.overground = overground + if not overground: + self.num_stations = int(row[4]) + self.num_lines = int(row[5] or '0') + self.num_light_lines = int(row[6] or '0') + self.num_interchanges = int(row[7] or '0') + else: + self.num_tram_lines = int(row[4] or '0') + self.num_trolleybus_lines = int(row[5] or '0') + self.num_bus_lines = int(row[6] or '0') + self.num_other_lines = int(row[7] or '0') # Aquiring list of networks and modes networks = None if len(row) <= 9 else row[9].split(':') @@ -885,7 +895,10 @@ class City: else: self.networks = set(filter(None, [x.strip() for x in networks[-1].split(';')])) if not networks or len(networks) < 2 or len(networks[0]) == 0: - self.modes = DEFAULT_MODES + if self.overground: + self.modes = DEFAULT_MODES_OVERGROUND + else: + self.modes = DEFAULT_MODES else: self.modes = set([x.strip() for x in networks[0].split(',')]) @@ -907,6 +920,28 @@ class City: self.errors = [] self.warnings = [] + def log_message(self, message, el): + if el: + tags = el.get('tags', {}) + message += ' ({} {}, "{}")'.format( + el['type'], el.get('id', el.get('ref')), + tags.get('name', tags.get('ref', ''))) + return message + + def warn(self, message, el=None): + msg = self.log_message(message, el) + self.warnings.append(msg) + + def error(self, message, el=None): + msg = self.log_message(message, el) + self.errors.append(msg) + + def error_if(self, is_error, message, el=None): + if is_error: + self.error(message, el) + else: + self.warn(message, el) + def contains(self, el): center = el_center(el) if center: @@ -938,48 +973,6 @@ class City: else: stop_area.append(el) - def get_validation_result(self): - result = { - 'name': self.name, - 'country': self.country, - 'continent': self.continent, - 'stations_expected': self.num_stations, - 'subwayl_expected': self.num_lines, - 'lightrl_expected': self.num_light_lines, - 'transfers_expected': self.num_interchanges, - 'stations_found': self.found_stations, - 'subwayl_found': self.found_lines, - 'lightrl_found': self.found_light_lines, - 'transfers_found': self.found_interchanges, - 'unused_entrances': self.unused_entrances, - 'networks': self.found_networks, - } - result['warnings'] = self.warnings - result['errors'] = self.errors - return result - - def log_message(self, message, el): - if el: - tags = el.get('tags', {}) - message += ' ({} {}, "{}")'.format( - el['type'], el.get('id', el.get('ref')), - tags.get('name', tags.get('ref', ''))) - return message - - def warn(self, message, el=None): - msg = self.log_message(message, el) - self.warnings.append(msg) - - def error(self, message, el=None): - msg = self.log_message(message, el) - self.errors.append(msg) - - def error_if(self, is_error, message, el=None): - if is_error: - self.error(message, el) - else: - self.warn(message, el) - def make_transfer(self, sag): transfer = set() for m in sag['members']: @@ -993,9 +986,6 @@ class City: if len(transfer) > 1: self.transfers.append(transfer) - def is_good(self): - return len(self.errors) == 0 - def extract_routes(self): # Extract stations processed_stop_areas = set() @@ -1070,6 +1060,45 @@ class City: def __iter__(self): return iter(self.routes.values()) + def is_good(self): + return len(self.errors) == 0 + + def get_validation_result(self): + result = { + 'name': self.name, + 'country': self.country, + 'continent': self.continent, + 'stations_found': self.found_stations, + 'transfers_found': self.found_interchanges, + 'unused_entrances': self.unused_entrances, + 'networks': self.found_networks, + } + if not self.overground: + result.update({ + 'subwayl_expected': self.num_lines, + 'lightrl_expected': self.num_light_lines, + 'subwayl_found': self.found_lines, + 'lightrl_found': self.found_light_lines, + 'stations_expected': self.num_stations, + 'transfers_expected': self.num_interchanges, + }) + else: + result.update({ + 'stations_expected': 0, + 'transfers_expected': 0, + 'busl_expected': self.num_bus_lines, + 'trolleybusl_expected': self.num_trolleybus_lines, + 'traml_expected': self.num_tram_lines, + 'otherl_expected': self.num_other_lines, + 'busl_found': self.found_bus_lines, + 'trolleybusl_found': self.found_trolleybus_lines, + 'traml_found': self.found_tram_lines, + 'otherl_found': self.found_other_lines, + }) + result['warnings'] = self.warnings + result['errors'] = self.errors + return result + def count_unused_entrances(self): global used_entrances stop_areas = set() @@ -1131,13 +1160,35 @@ class City: if t not in have_return: self.warn('Route does not have a return direction', rel) + def validate_lines(self): + self.found_light_lines = len([x for x in self.routes.values() if x.mode != 'subway']) + self.found_lines = len(self.routes) - self.found_light_lines + if self.found_lines != self.num_lines: + self.error('Found {} subway lines, expected {}'.format( + self.found_lines, self.num_lines)) + if self.found_light_lines != self.num_light_lines: + self.error('Found {} light rail lines, expected {}'.format( + self.found_light_lines, self.num_light_lines)) + + def validate_overground_lines(self): + self.found_tram_lines = len([x for x in self.routes.values() if x.mode == 'tram']) + self.found_bus_lines = len([x for x in self.routes.values() if x.mode == 'bus']) + self.found_trolleybus_lines = len([x for x in self.routes.values() + if x.mode == 'trolleybus']) + self.found_other_lines = len([x for x in self.routes.values() + if x.mode not in ('bus', 'trolleybus', 'tram')]) + if self.found_tram_lines != self.num_tram_lines: + self.warn('Found {} tram lines, expected {}'.format( + self.found_tram_lines, self.num_tram_lines)) + def validate(self): networks = Counter() self.found_stations = 0 unused_stations = set(self.station_ids) for rmaster in self.routes.values(): networks[str(rmaster.network)] += 1 - self.check_return_routes(rmaster) + if not self.overground: + self.check_return_routes(rmaster) route_stations = set() for sa in rmaster.stop_areas(): route_stations.add(sa.transfer or sa.id) @@ -1148,29 +1199,26 @@ class City: self.warn('{} unused stations: {}'.format( self.unused_stations, format_elid_list(unused_stations))) self.count_unused_entrances() - - self.found_light_lines = len([x for x in self.routes.values() if x.mode != 'subway']) - self.found_lines = len(self.routes) - self.found_light_lines - if self.found_lines != self.num_lines: - self.error('Found {} subway lines, expected {}'.format( - self.found_lines, self.num_lines)) - if self.found_light_lines != self.num_light_lines: - self.error('Found {} light rail lines, expected {}'.format( - self.found_light_lines, self.num_light_lines)) - - if self.found_stations != self.num_stations: - msg = 'Found {} stations in routes, expected {}'.format( - self.found_stations, self.num_stations) - self.error_if(not (0 <= (self.num_stations - self.found_stations) / self.num_stations <= - ALLOWED_STATIONS_MISMATCH), msg) - self.found_interchanges = len(self.transfers) - if self.found_interchanges != self.num_interchanges: - msg = 'Found {} interchanges, expected {}'.format( - self.found_interchanges, self.num_interchanges) - self.error_if(self.num_interchanges != 0 and not - ((self.num_interchanges - self.found_interchanges) / - self.num_interchanges <= ALLOWED_TRANSFERS_MISMATCH), msg) + + if self.overground: + self.validate_overground_lines() + else: + self.validate_lines() + + if self.found_stations != self.num_stations: + msg = 'Found {} stations in routes, expected {}'.format( + self.found_stations, self.num_stations) + self.error_if(not (0 <= + (self.num_stations - self.found_stations) / self.num_stations <= + ALLOWED_STATIONS_MISMATCH), msg) + + if self.found_interchanges != self.num_interchanges: + msg = 'Found {} interchanges, expected {}'.format( + self.found_interchanges, self.num_interchanges) + self.error_if(self.num_interchanges != 0 and not + ((self.num_interchanges - self.found_interchanges) / + self.num_interchanges <= ALLOWED_TRANSFERS_MISMATCH), msg) self.found_networks = len(networks) if len(networks) > max(1, len(self.networks)): @@ -1218,8 +1266,9 @@ def get_unused_entrances_geojson(elements): return {'type': 'FeatureCollection', 'features': features} -def download_cities(): - url = 'https://docs.google.com/spreadsheets/d/{}/export?format=csv'.format(SPREADSHEET_ID) +def download_cities(overground=False): + url = 'https://docs.google.com/spreadsheets/d/{}/export?format=csv{}'.format( + SPREADSHEET_ID, '&gid=1881416409' if overground else '') response = urllib.request.urlopen(url) if response.getcode() != 200: raise Exception('Failed to download cities spreadsheet: HTTP {}'.format(response.getcode())) @@ -1230,7 +1279,7 @@ def download_cities(): cities = [] for row in r: if len(row) > 8 and row[8]: - cities.append(City(row)) + cities.append(City(row, overground)) if row[0].strip() in names: logging.warning('Duplicate city name in the google spreadsheet: %s', row[0]) names.add(row[0].strip()) diff --git a/v2h_templates.py b/v2h_templates.py index 36f7487..c4b8e81 100644 --- a/v2h_templates.py +++ b/v2h_templates.py @@ -114,8 +114,15 @@ COUNTRY_HEADER = ''' +{?subways} +{end}{?overground} + + + + +{end} @@ -125,13 +132,20 @@ COUNTRY_HEADER = ''' COUNTRY_CITY = ''' +{?subways} +{end}{?overground} + + + + +{end} -
CitySubway Lines Light Rail LinesTram LinesBus LinesT-Bus LinesOther LinesStations Interchanges Unused Entrances
{city}{?yaml} Y{end}{?json} J{end}sub: {subwayl_found} / {subwayl_expected} lr: {lightrl_found} / {lightrl_expected}t: {traml_found} / {traml_expected}b: {busl_found} / {busl_expected}tb: {trolleybusl_found} / {trolleybusl_expected}o: {otherl_found} / {otherl_expected}st: {stations_found} / {stations_expected} int: {transfers_found} / {transfers_expected} e: {unused_entrances}
+
{errors}
diff --git a/validation_to_html.py b/validation_to_html.py index 406f2e5..76faf52 100755 --- a/validation_to_html.py +++ b/validation_to_html.py @@ -53,9 +53,11 @@ class CityData: for k in self.data: s = s.replace('{'+k+'}', str(self.data[k])) s = s.replace('{slug}', self.slug or '') - for k in ('subwayl', 'lightrl', 'stations', 'transfers'): - s = s.replace('{='+k+'}', - test_eq(self.data[k+'_found'], self.data[k+'_expected'])) + for k in ('subwayl', 'lightrl', 'stations', 'transfers', 'busl', + 'trolleybusl', 'traml', 'otherl'): + if k+'_expected' in self.data: + s = s.replace('{='+k+'}', + test_eq(self.data[k+'_found'], self.data[k+'_expected'])) s = s.replace('{=cities}', test_eq(self.data['good_cities'], self.data['total_cities'])) s = s.replace('{=entrances}', test_eq(self.data['unused_entrances'], 0)) @@ -71,7 +73,7 @@ def tmpl(s, data=None, **kwargs): for k, v in kwargs.items(): if v is not None: s = s.replace('{'+k+'}', str(v)) - s = re.sub(r'\{\?'+k+r'\}(.+?)\{end\}', r'\1' if v else '', s) + s = re.sub(r'\{\?'+k+r'\}(.+?)\{end\}', r'\1' if v else '', s, flags=re.DOTALL) s = s.replace('{date}', date) google_url = 'https://docs.google.com/spreadsheets/d/{}/edit?usp=sharing'.format(SPREADSHEET_ID) s = s.replace('{google}', google_url) @@ -119,6 +121,7 @@ for c in data.values(): c_by_c[c.continent].add(c.country) world = sum(continents.values(), CityData()) +overground = 'traml_expected' in next(iter(data.values())).data date = datetime.datetime.now().strftime('%d.%m.%Y %H:%M') path = '.' if len(sys.argv) < 3 else sys.argv[2] index = open(os.path.join(path, 'index.html'), 'w', encoding='utf-8') @@ -131,7 +134,8 @@ for continent in sorted(continents.keys()): content += tmpl(INDEX_COUNTRY, countries[country], file=country_file_name, country=country, continent=continent) country_file = open(os.path.join(path, country_file_name), 'w', encoding='utf-8') - country_file.write(tmpl(COUNTRY_HEADER, country=country, continent=continent)) + country_file.write(tmpl(COUNTRY_HEADER, country=country, continent=continent, + overground=overground, subways=not overground)) for name, city in sorted(data.items()): if city.country == country: file_base = os.path.join(path, city.slug) @@ -142,8 +146,8 @@ for continent in sorted(continents.keys()): w = '
'.join([osm_links(esc(w)) for w in city.warnings]) country_file.write(tmpl(COUNTRY_CITY, city, city=name, country=country, continent=continent, - yaml=yaml_file, json=json_file, - errors=e, warnings=w)) + yaml=yaml_file, json=json_file, subways=not overground, + errors=e, warnings=w, overground=overground)) country_file.write(tmpl(COUNTRY_FOOTER, country=country, continent=continent)) country_file.close() index.write(tmpl(INDEX_CONTINENT, continents[continent],