diff --git a/web/app/borders_api.py b/web/app/borders_api.py index c04c46d..ffe277b 100755 --- a/web/app/borders_api.py +++ b/web/app/borders_api.py @@ -413,31 +413,11 @@ def find_osm_borders(): def copy_from_osm(): osm_id = int(request.args.get('id')) name = request.args.get('name') - name_sql = f"'{name}'" if name else "'name'" - borders_table = config.BORDERS_TABLE - osm_table = config.OSM_TABLE - with g.conn.cursor() as cursor: - # Check if this id already in use - cursor.execute(f"SELECT id FROM {borders_table} WHERE id = %s", - (osm_id,)) - rec = cursor.fetchone() - if rec and rec[0]: - return jsonify(status=f"Region with id={osm_id} already exists") - cursor.execute(f""" - INSERT INTO {borders_table} (id, geom, name, modified, count_k) - SELECT osm_id, way, {name_sql}, now(), -1 - FROM {osm_table} - WHERE osm_id = %s - """, (osm_id,) - ) - assign_region_to_lowest_parent(osm_id) - warnings = [] - try: - update_border_mwm_size_estimation(g.conn, osm_id) - except Exception as e: - warnings.append(str(e)) + success = copy_region_from_osm(g.conn, osm_id, name) + if not success: + return jsonify(status=f"Region with id={osm_id} already exists") g.conn.commit() - return jsonify(status='ok', warnings=warnings) + return jsonify(status='ok') @app.route('/rename') @@ -983,7 +963,7 @@ def border(): @app.route('/start_over') def start_over(): try: - warnings = create_countries_initial_structure(g.conn) + create_countries_initial_structure(g.conn) except CountryStructureException as e: return jsonify(status=str(e)) @@ -991,7 +971,7 @@ def start_over(): with g.conn.cursor() as cursor: cursor.execute(f"DELETE FROM {autosplit_table}") g.conn.commit() - return jsonify(status='ok', warnings=warnings[:10]) + return jsonify(status='ok') if __name__ == '__main__': diff --git a/web/app/borders_api_utils.py b/web/app/borders_api_utils.py index 6b6f967..0421b33 100644 --- a/web/app/borders_api_utils.py +++ b/web/app/borders_api_utils.py @@ -3,13 +3,18 @@ import json from flask import g, jsonify -import config +from config import ( + AUTOSPLIT_TABLE as autosplit_table, + BORDERS_TABLE as main_borders_table, + OSM_TABLE as osm_table, +) from auto_split import split_region -from countries_structure import get_osm_border_name_by_osm_id from subregions import ( + get_parent_region_id, get_region_country, get_subregions_info, is_administrative_region, + update_border_mwm_size_estimation, ) @@ -19,11 +24,10 @@ def geom_intersects_bbox_sql(xmin, ymin, xmax, ymax): def fetch_borders(**kwargs): - borders_table = kwargs.get('table', config.BORDERS_TABLE) + borders_table = kwargs.get('table', main_borders_table) simplify = kwargs.get('simplify', 0) where_clause = kwargs.get('where_clause', '1=1') only_leaves = kwargs.get('only_leaves', True) - osm_table = config.OSM_TABLE geom = (f'ST_SimplifyPreserveTopology(geom, {simplify})' if simplify > 0 else 'geom') leaves_filter = (f""" AND id NOT IN (SELECT parent_id FROM {borders_table} @@ -104,8 +108,7 @@ def get_subregions_for_preview(region_ids, next_level): def get_subregions_one_for_preview(region_id, next_level): - osm_table = config.OSM_TABLE - borders_table = config.BORDERS_TABLE + borders_table = main_borders_table with g.conn.cursor() as cursor: # We use ST_SimplifyPreserveTopology, since ST_Simplify would give NULL # for very little regions. @@ -137,7 +140,6 @@ def get_clusters_for_preview(region_ids, next_level, thresholds): def get_clusters_for_preview_one(region_id, next_level, mwm_size_thr): - autosplit_table = config.AUTOSPLIT_TABLE where_clause = f""" osm_border_id = %s AND mwm_size_thr = %s @@ -190,47 +192,56 @@ def divide_into_clusters_preview(region_ids, next_level, mwm_size_thr): def divide_into_subregions(region_ids, next_level): for region_id in region_ids: - divide_into_subregions_one(region_id, next_level) + divide_region_into_subregions(g.conn, region_id, next_level) g.conn.commit() return jsonify(status='ok') -def divide_into_subregions_one(region_id, next_level): - borders_table = config.BORDERS_TABLE - osm_table = config.OSM_TABLE - subregions = get_subregions_info(g.conn, region_id, borders_table, +def divide_region_into_subregions(conn, region_id, next_level): + """Divides a region into subregions of specified admin level. + Returns the list of added subregion ids. + """ + borders_table = main_borders_table + subregions = get_subregions_info(conn, region_id, borders_table, next_level, need_cities=False) - with g.conn.cursor() as cursor: - is_admin_region = is_administrative_region(g.conn, region_id) + if not subregions: + return [] + with conn.cursor() as cursor: + subregion_ids_str = ','.join(str(x) for x in subregions.keys()) + cursor.execute(f""" + SELECT id + FROM {borders_table} + WHERE id IN ({subregion_ids_str}) + """ + ) + occupied_ids = [rec[0] for rec in cursor] + ids_to_insert = set(subregions.keys()) - set(occupied_ids) + if not ids_to_insert: + return [] + + is_admin_region = is_administrative_region(conn, region_id) + if is_admin_region: - for subregion_id, data in subregions.items(): - cursor.execute(f""" - INSERT INTO {borders_table} - (id, geom, name, parent_id, modified, count_k, mwm_size_est) - SELECT osm_id, way, name, %s, now(), -1, {data['mwm_size_est']} - FROM {osm_table} - WHERE osm_id = %s - """, (region_id, subregion_id) - ) + parent_id = region_id else: - for subregion_id, data in subregions.items(): - cursor.execute(f""" - INSERT INTO {borders_table} - (id, geom, name, parent_id, modified, count_k, mwm_size_est) - SELECT osm_id, way, name, - (SELECT parent_id FROM {borders_table} WHERE id = %s), - now(), -1, {data['mwm_size_est']} - FROM {osm_table} - WHERE osm_id = %s - """, (region_id, subregion_id) - ) + parent_id = get_parent_region_id(conn, region_id) + + for subregion_id in ids_to_insert: + mwm_size_est = subregions[subregion_id]['mwm_size_est'] + cursor.execute(f""" + INSERT INTO {borders_table} + (id, geom, name, parent_id, modified, count_k, mwm_size_est) + SELECT osm_id, way, name, {parent_id}, now(), -1, {mwm_size_est} + FROM {osm_table} + WHERE osm_id = %s""", (subregion_id,) + ) + if not is_admin_region: cursor.execute(f"DELETE FROM {borders_table} WHERE id = %s", (region_id,)) - g.conn.commit() + return ids_to_insert def divide_into_clusters(region_ids, next_level, mwm_size_thr): - borders_table = config.BORDERS_TABLE - autosplit_table = config.AUTOSPLIT_TABLE + borders_table = main_borders_table cursor = g.conn.cursor() insert_cursor = g.conn.cursor() for region_id in region_ids: @@ -282,22 +293,22 @@ def divide_into_clusters(region_ids, next_level, mwm_size_thr): def get_free_id(): with g.conn.cursor() as cursor: - borders_table = config.BORDERS_TABLE + borders_table = main_borders_table cursor.execute(f"SELECT min(id) FROM {borders_table} WHERE id < -1000000000") min_id = cursor.fetchone()[0] free_id = min_id - 1 if min_id else -1_000_000_001 return free_id -def assign_region_to_lowest_parent(region_id): +def assign_region_to_lowest_parent(conn, region_id): """Lowest parent is the region with lowest (maximum by absolute value) admin_level containing given region.""" pot_parents = find_potential_parents(region_id) if pot_parents: # potential_parents are sorted by area ascending parent_id = pot_parents[0]['properties']['id'] - borders_table = config.BORDERS_TABLE - with g.conn.cursor() as cursor: + borders_table = main_borders_table + with conn.cursor() as cursor: cursor.execute(f""" UPDATE {borders_table} SET parent_id = %s @@ -309,7 +320,7 @@ def assign_region_to_lowest_parent(region_id): def create_or_update_region(region, free_id): - borders_table = config.BORDERS_TABLE + borders_table = main_borders_table with g.conn.cursor() as cursor: if region['id'] < 0: if not free_id: @@ -323,7 +334,7 @@ def create_or_update_region(region, free_id): """, (region_id, region['name'], region['disabled'], region['wkt']) ) - assign_region_to_lowest_parent(region_id) + assign_region_to_lowest_parent(g.conn, region_id) return region_id else: cursor.execute(f"SELECT count(1) FROM {borders_table} WHERE id = %s", @@ -346,8 +357,7 @@ def create_or_update_region(region, free_id): def find_potential_parents(region_id): - borders_table = config.BORDERS_TABLE - osm_table = config.OSM_TABLE + borders_table = main_borders_table p_geogr = "geography(p.geom)" c_geogr = "geography(c.geom)" query = f""" @@ -380,3 +390,39 @@ def find_potential_parents(region_id): } parents.append(feature) return parents + + +def copy_region_from_osm(conn, region_id, name=None, parent_id='not_passed'): + borders_table = main_borders_table + with conn.cursor() as cursor: + # Check if this id already in use + cursor.execute(f"SELECT id FROM {borders_table} WHERE id = %s", + (region_id,)) + if cursor.rowcount > 0: + return False + + name_expr = f"'{name}'" if name else "name" + parent_id_expr = f"{parent_id}" if isinstance(parent_id, int) else "NULL" + cursor.execute(f""" + INSERT INTO {borders_table} + (id, geom, name, parent_id, modified, count_k) + SELECT osm_id, way, {name_expr}, {parent_id_expr}, now(), -1 + FROM {osm_table} + WHERE osm_id = %s + """, (region_id,) + ) + if parent_id == 'not_passed': + assign_region_to_lowest_parent(conn, region_id) + update_border_mwm_size_estimation(conn, region_id) + return True + + +def get_osm_border_name_by_osm_id(conn, osm_id): + with conn.cursor() as cursor: + cursor.execute(f""" + SELECT name FROM {osm_table} + WHERE osm_id = %s + """, (osm_id,)) + rec = cursor.fetchone() + return rec[0] if rec else None + diff --git a/web/app/countries_structure.py b/web/app/countries_structure.py index 0a0e5fe..b9f7bda 100644 --- a/web/app/countries_structure.py +++ b/web/app/countries_structure.py @@ -1,3 +1,8 @@ +from borders_api_utils import ( + copy_region_from_osm, + divide_region_into_subregions, + get_osm_border_name_by_osm_id, +) from config import ( BORDERS_TABLE as borders_table, OSM_TABLE as osm_table @@ -16,80 +21,25 @@ class CountryStructureException(Exception): def _clear_borders(conn): with conn.cursor() as cursor: cursor.execute(f"DELETE FROM {borders_table}") - conn.commit() - - -def _find_subregions(conn, osm_ids, next_level, regions): - """Return subregions of level 'next_level' for regions with osm_ids.""" - subregion_ids = [] - for osm_id in osm_ids: - more_subregions = get_subregions_info(conn, osm_id, borders_table, - next_level, need_cities=False) - for subregion_id, subregion_data in more_subregions.items(): - region_data = regions.setdefault(subregion_id, {}) - region_data['name'] = subregion_data['name'] - region_data['mwm_size_est'] = subregion_data['mwm_size_est'] - region_data['parent_id'] = osm_id - subregion_ids.append(subregion_id) - return subregion_ids - - -def _create_regions(conn, osm_ids, regions): - if not osm_ids: - return - osm_ids = list(osm_ids) # to ensure order - sql_values = ','.join( - f'({osm_id},' - '%s,' - f"{regions[osm_id].get('parent_id', 'NULL')}," - f"{regions[osm_id].get('mwm_size_est', 'NULL')}," - f'(SELECT way FROM {osm_table} WHERE osm_id={osm_id}),' - 'now())' - for osm_id in osm_ids - ) - with conn.cursor() as cursor: - cursor.execute(f""" - INSERT INTO {borders_table} (id, name, parent_id, mwm_size_est, - geom, modified) - VALUES {sql_values} - """, tuple(regions[osm_id]['name'] for osm_id in osm_ids) - ) def _make_country_structure(conn, country_osm_id): - regions = {} # osm_id: { 'name': name, - # 'mwm_size_est': size, - # 'parent_id': parent_id } - country_name = get_osm_border_name_by_osm_id(conn, country_osm_id) - country_data = regions.setdefault(country_osm_id, {}) - country_data['name'] = country_name - # TODO: country_data['mwm_size_est'] = ... - _create_regions(conn, [country_osm_id], regions) + copy_region_from_osm(conn, country_osm_id, parent_id=None) if country_initial_levels.get(country_name): admin_levels = country_initial_levels[country_name] prev_admin_levels = [2] + admin_levels[:-1] - prev_region_ids = [country_osm_id] + prev_level_region_ids = [country_osm_id] for admin_level, prev_level in zip(admin_levels, prev_admin_levels): - if not prev_region_ids: - raise CountryStructureException( - f"Empty prev_region_ids at {country_name}, " - f"AL={admin_level}, prev-AL={prev_level}" - ) - subregion_ids = _find_subregions(conn, prev_region_ids, - admin_level, regions) - _create_regions(conn, subregion_ids, regions) - prev_region_ids = subregion_ids - warning = None - if len(regions) == 1: - try: - update_border_mwm_size_estimation(conn, country_osm_id) - except Exception as e: - warning = str(e) - return warning + current_level_region_ids = [] + for region_id in prev_level_region_ids: + subregion_ids = divide_region_into_subregions( + conn, region_id, admin_level) + current_level_region_ids.extend(subregion_ids) + prev_level_region_ids = current_level_region_ids def create_countries_initial_structure(conn): @@ -99,30 +49,13 @@ def create_countries_initial_structure(conn): cursor.execute(f""" SELECT osm_id, name FROM {osm_table} - WHERE admin_level = 2 and name != 'Ukraine' + WHERE admin_level = 2 """ ) - warnings = [] - for rec in cursor: - warning = _make_country_structure(conn, rec[0]) - if warning: - warnings.append(warning) + for country_osm_id, *_ in cursor: + _make_country_structure(conn, country_osm_id) conn.commit() - return warnings - - -def get_osm_border_name_by_osm_id(conn, osm_id): - with conn.cursor() as cursor: - cursor.execute(f""" - SELECT name FROM {osm_table} - WHERE osm_id = %s - """, (osm_id,)) - rec = cursor.fetchone() - if not rec: - raise CountryStructureException( - f'Not found region with osm_id="{osm_id}"' - ) - return rec[0] + return def _get_country_osm_id_by_name(conn, name): @@ -137,4 +70,5 @@ def _get_country_osm_id_by_name(conn, name): rec = cursor.fetchone() if not rec: raise CountryStructureException(f'Not found country "{name}"') - return int(rec[0]) + return rec[0] + diff --git a/web/app/subregions.py b/web/app/subregions.py index b9db6fb..0a281cf 100644 --- a/web/app/subregions.py +++ b/web/app/subregions.py @@ -88,6 +88,8 @@ def _add_population_data(conn, subregions, need_cities): def _add_mwm_size_estimation(subregions): + if not subregions: + return subregions_sorted = [ ( s_id, @@ -245,7 +247,7 @@ def get_similar_regions(conn, region_id, only_leaves=False): if item['admin_level'] == admin_level: similar_region_ids.append(item['id']) elif item['admin_level'] < admin_level: - children = find_osm_child_regions(item['id']) + children = find_osm_child_regions(conn, item['id']) for ch in children: q.put(ch) if only_leaves: