diff --git a/web/app/auto_split_debug.py b/web/app/auto_split_debug.py index 440cad9..f8e9de7 100644 --- a/web/app/auto_split_debug.py +++ b/web/app/auto_split_debug.py @@ -5,7 +5,7 @@ from auto_split import ( DisjointClusterUnion, get_union_sql, ) -from countries_structure import ( +from subregions import ( get_region_full_name, ) diff --git a/web/app/borders_api.py b/web/app/borders_api.py index 56af1e1..287427f 100755 --- a/web/app/borders_api.py +++ b/web/app/borders_api.py @@ -1,6 +1,5 @@ #!/usr/bin/python3.6 import io -import itertools import re import sys, traceback import zipfile @@ -18,15 +17,10 @@ from flask_compress import Compress import psycopg2 import config -from auto_split import split_region +from borders_api_utils import * from countries_structure import ( CountryStructureException, create_countries_initial_structure, - get_osm_border_name_by_osm_id, - get_region_country, - get_region_full_name, - get_similar_regions, - is_administrative_region, ) from osm_xml import ( borders_from_xml, @@ -34,7 +28,11 @@ from osm_xml import ( lines_to_xml, ) from subregions import ( - get_subregions_info, + get_child_region_ids, + get_parent_region_id, + get_region_full_name, + get_similar_regions, + is_administrative_region, update_border_mwm_size_estimation, ) @@ -58,99 +56,29 @@ def send_js(path): return send_from_directory('static/', path) abort(404) + @app.before_request def before_request(): g.conn = psycopg2.connect(config.CONNECTION) + @app.teardown_request def teardown(exception): conn = getattr(g, 'conn', None) if conn is not None: conn.close() + @app.route('/') @app.route('/index.html') def index(): return render_template('index.html') + @app.route('/stat.html') def stat(): return render_template('stat.html') -def fetch_borders(**kwargs): - table = kwargs.get('table', config.TABLE) - simplify = kwargs.get('simplify', 0) - where_clause = kwargs.get('where_clause', '1=1') - only_leaves = kwargs.get('only_leaves', True) - osm_table = config.OSM_TABLE - geom = (f'ST_SimplifyPreserveTopology(geom, {simplify})' - if simplify > 0 else 'geom') - leaves_filter = (f""" AND id NOT IN (SELECT parent_id FROM {table} - WHERE parent_id IS NOT NULL)""" - if only_leaves else '') - query = f""" - SELECT name, geometry, nodes, modified, disabled, count_k, cmnt, - (CASE WHEN area = 'NaN'::DOUBLE PRECISION THEN 0 ELSE area END) AS area, - id, admin_level, parent_id, parent_name, parent_admin_level, - mwm_size_est - FROM ( - SELECT name, - ST_AsGeoJSON({geom}, 7) as geometry, - ST_NPoints(geom) AS nodes, - modified, - disabled, - count_k, - cmnt, - round(ST_Area(geography(geom))) AS area, - id, - ( SELECT admin_level FROM {osm_table} - WHERE osm_id = t.id - ) AS admin_level, - parent_id, - ( SELECT name FROM {table} - WHERE id = t.parent_id - ) AS parent_name, - ( SELECT admin_level FROM {osm_table} - WHERE osm_id = (SELECT parent_id FROM {table} WHERE id = t.id) - ) AS parent_admin_level, - mwm_size_est - FROM {table} t - WHERE ({where_clause}) {leaves_filter} - ) q - ORDER BY area DESC - """ - cur = g.conn.cursor() - cur.execute(query) - borders = [] - for rec in cur: - region_id = rec[8] - country_id, country_name = get_region_country(g.conn, region_id) - props = { 'name': rec[0] or '', 'nodes': rec[2], 'modified': rec[3], - 'disabled': rec[4], 'count_k': rec[5], - 'comment': rec[6], - 'area': rec[7], - 'id': region_id, - 'admin_level': rec[9], - 'parent_id': rec[10], - 'parent_name': rec[11], - 'parent_admin_level': rec[12], - 'country_id': country_id, - 'country_name': country_name, - 'mwm_size_est': rec[13] - } - feature = {'type': 'Feature', - 'geometry': json.loads(rec[1]), - 'properties': props - } - borders.append(feature) - return borders - -def simplify_level_to_postgis_value(simplify_level): - return ( - 0.1 if simplify_level == '2' - else 0.01 if simplify_level == '1' - else 0 - ) @app.route('/bbox') def query_bbox(): @@ -176,6 +104,7 @@ def query_bbox(): geojson={'type': 'FeatureCollection', 'features': borders} ) + @app.route('/small') def query_small_in_bbox(): xmin = request.args.get('xmin') @@ -205,6 +134,7 @@ def query_small_in_bbox(): 'lon': float(rec[2]), 'lat': float(rec[3]) }) return jsonify(features=result) + @app.route('/config') def get_server_configuration(): osm = False @@ -239,6 +169,7 @@ def get_server_configuration(): backup=backup, mwm_size_thr=config.MWM_SIZE_THRESHOLD) + @app.route('/search') def search(): query = request.args.get('q') @@ -255,6 +186,7 @@ def search(): return jsonify(bounds=[rec[0], rec[1], rec[2], rec[3]]) return jsonify(status='not found') + @app.route('/split') def split(): if config.READONLY: @@ -315,6 +247,7 @@ def split(): g.conn.commit() return jsonify(status='ok', warnings=warnings) + @app.route('/join') def join_borders(): if config.READONLY: @@ -343,33 +276,15 @@ def join_borders(): g.conn.commit() return jsonify(status='ok') -def get_parent_region_id(region_id): - cursor = g.conn.cursor() - cursor.execute(f""" - SELECT parent_id FROM {config.TABLE} WHERE id = %s - """, (region_id,)) - rec = cursor.fetchone() - parent_id = int(rec[0]) if rec and rec[0] is not None else None - return parent_id - -def get_child_region_ids(region_id): - cursor = g.conn.cursor() - cursor.execute(f""" - SELECT id FROM {config.TABLE} WHERE parent_id = %s - """, (region_id,)) - child_ids = [] - for rec in cursor: - child_ids.append(int(rec[0])) - return child_ids @app.route('/join_to_parent') def join_to_parent(): """Find all descendants of a region and remove them starting - from the lowerst hierarchical level to not violate 'parent_id' + from the lowest hierarchical level to not violate 'parent_id' foreign key constraint (which is probably not in ON DELETE CASCADE mode) """ region_id = int(request.args.get('id')) - parent_id = get_parent_region_id(region_id) + parent_id = get_parent_region_id(g.conn, region_id) if not parent_id: return jsonify(status=f"Region {region_id} does not exist or has no parent") cursor = g.conn.cursor() @@ -378,21 +293,22 @@ def join_to_parent(): while True: parent_ids = descendants[-1] child_ids = list(itertools.chain.from_iterable( - get_child_region_ids(x) for x in parent_ids + get_child_region_ids(g.conn, parent_id) for parent_id in parent_ids )) if child_ids: descendants.append(child_ids) else: break while len(descendants) > 1: - lowerst_ids = descendants.pop() - ids_str = ','.join(str(x) for x in lowerst_ids) + lowest_ids = descendants.pop() + ids_str = ','.join(str(x) for x in lowest_ids) cursor.execute(f""" DELETE FROM {config.TABLE} WHERE id IN ({ids_str})""" ) g.conn.commit() return jsonify(status='ok') + @app.route('/set_parent') def set_parent(): region_id = int(request.args.get('id')) @@ -407,6 +323,7 @@ def set_parent(): g.conn.commit() return jsonify(status='ok') + @app.route('/point') def find_osm_borders(): lat = request.args.get('lat') @@ -430,6 +347,7 @@ def find_osm_borders(): result.append(border) return jsonify(borders=result) + @app.route('/from_osm') def copy_from_osm(): if config.READONLY: @@ -452,7 +370,7 @@ def copy_from_osm(): WHERE osm_id = %s """, (osm_id,) ) - assign_region_to_lowerst_parent(osm_id) + assign_region_to_lowest_parent(osm_id) warnings = [] try: update_border_mwm_size_estimation(g.conn, osm_id) @@ -461,6 +379,7 @@ def copy_from_osm(): g.conn.commit() return jsonify(status='ok', warnings=warnings) + @app.route('/rename') def set_name(): if config.READONLY: @@ -474,6 +393,7 @@ def set_name(): g.conn.commit() return jsonify(status='ok') + @app.route('/delete') def delete_border(): if config.READONLY: @@ -484,6 +404,7 @@ def delete_border(): g.conn.commit() return jsonify(status='ok') + @app.route('/disable') def disable_border(): if config.READONLY: @@ -495,6 +416,7 @@ def disable_border(): g.conn.commit() return jsonify(status='ok') + @app.route('/enable') def enable_border(): if config.READONLY: @@ -506,6 +428,7 @@ def enable_border(): g.conn.commit() return jsonify(status='ok') + @app.route('/comment', methods=['POST']) def update_comment(): region_id = int(request.form['id']) @@ -545,92 +468,6 @@ def divide_preview(): else: return divide_into_subregions_preview(region_ids, next_level) -def get_subregions_for_preview(region_ids, next_level): - subregions = list(itertools.chain.from_iterable( - get_subregions_one_for_preview(region_id, next_level) - for region_id in region_ids - )) - return subregions - -def get_subregions_one_for_preview(region_id, next_level): - osm_table = config.OSM_TABLE - table = config.TABLE - cur = g.conn.cursor() - # We use ST_SimplifyPreserveTopology, since ST_Simplify would give NULL - # for very little regions. - cur.execute(f""" - SELECT name, - ST_AsGeoJSON(ST_SimplifyPreserveTopology(way, 0.01)) as way, - osm_id - FROM {osm_table} - WHERE ST_Contains( - (SELECT geom FROM {table} WHERE id = %s), way - ) - AND admin_level = %s - """, (region_id, next_level) - ) - subregions = [] - for rec in cur: - feature = {'type': 'Feature', 'geometry': json.loads(rec[1]), - 'properties': {'name': rec[0]}} - subregions.append(feature) - return subregions - -def get_clusters_for_preview(region_ids, next_level, thresholds): - clusters = list(itertools.chain.from_iterable( - get_clusters_for_preview_one(region_id, next_level, thresholds) - for region_id in region_ids - )) - return clusters - -def get_clusters_for_preview_one(region_id, next_level, mwm_size_thr): - autosplit_table = config.AUTOSPLIT_TABLE - cursor = g.conn.cursor() - where_clause = f""" - osm_border_id = %s - AND mwm_size_thr = %s - """ - splitting_sql_params = (region_id, mwm_size_thr) - cursor.execute(f""" - SELECT 1 FROM {autosplit_table} - WHERE {where_clause} - """, splitting_sql_params - ) - if cursor.rowcount == 0: - split_region(g.conn, region_id, next_level, mwm_size_thr) - - cursor.execute(f""" - SELECT subregion_ids[1], - ST_AsGeoJSON(ST_SimplifyPreserveTopology(geom, 0.01)) as way - FROM {autosplit_table} - WHERE {where_clause} - """, splitting_sql_params - ) - clusters = [] - for rec in cursor: - cluster = { - 'type': 'Feature', - 'geometry': json.loads(rec[1]), - 'properties': {'osm_id': int(rec[0])} - } - clusters.append(cluster) - return clusters - -def divide_into_subregions_preview(region_ids, next_level): - subregions = get_subregions_for_preview(region_ids, next_level) - return jsonify( - status='ok', - subregions={'type': 'FeatureCollection', 'features': subregions} - ) - -def divide_into_clusters_preview(region_ids, next_level, mwm_size_thr): - subregions = get_subregions_for_preview(region_ids, next_level) - clusters = get_clusters_for_preview(region_ids, next_level, mwm_size_thr) - return jsonify( - status='ok', - subregions={'type': 'FeatureCollection', 'features': subregions}, - clusters={'type': 'FeatureCollection', 'features': clusters} - ) @app.route('/divide') def divide(): @@ -662,93 +499,6 @@ def divide(): else: return divide_into_subregions(region_ids, next_level) -def divide_into_subregions(region_ids, next_level): - for region_id in region_ids: - divide_into_subregions_one(region_id, next_level) - g.conn.commit() - return jsonify(status='ok') - -def divide_into_subregions_one(region_id, next_level): - table = config.TABLE - osm_table = config.OSM_TABLE - subregions = get_subregions_info(g.conn, region_id, table, - next_level, need_cities=False) - cursor = g.conn.cursor() - is_admin_region = is_administrative_region(g.conn, region_id) - if is_admin_region: - for subregion_id, data in subregions.items(): - cursor.execute(f""" - INSERT INTO {table} - (id, geom, name, parent_id, modified, count_k, mwm_size_est) - SELECT osm_id, way, name, %s, now(), -1, {data['mwm_size_est']} - FROM {osm_table} - WHERE osm_id = %s - """, (region_id, subregion_id) - ) - else: - for subregion_id, data in subregions.items(): - cursor.execute(f""" - INSERT INTO {table} - (id, geom, name, parent_id, modified, count_k, mwm_size_est) - SELECT osm_id, way, name, - (SELECT parent_id FROM {table} WHERE id = %s), - now(), -1, {data['mwm_size_est']} - FROM {osm_table} - WHERE osm_id = %s - """, (region_id, subregion_id) - ) - cursor.execute(f"DELETE FROM {table} WHERE id = %s", (region_id,)) - -def divide_into_clusters(region_ids, next_level, mwm_size_thr): - table = config.TABLE - autosplit_table = config.AUTOSPLIT_TABLE - cursor = g.conn.cursor() - insert_cursor = g.conn.cursor() - for region_id in region_ids: - cursor.execute(f"SELECT name FROM {table} WHERE id = %s", (region_id,)) - base_name = cursor.fetchone()[0] - - where_clause = f""" - osm_border_id = %s - AND mwm_size_thr = %s - """ - splitting_sql_params = (region_id, mwm_size_thr) - cursor.execute(f""" - SELECT 1 FROM {autosplit_table} - WHERE {where_clause} - """, splitting_sql_params - ) - if cursor.rowcount == 0: - split_region(g.conn, region_id, next_level, mwm_size_thr) - - free_id = get_free_id() - counter = 0 - cursor.execute(f""" - SELECT subregion_ids - FROM {autosplit_table} WHERE {where_clause} - """, splitting_sql_params - ) - if cursor.rowcount == 1: - continue - for rec in cursor: - subregion_ids = rec[0] - cluster_id = subregion_ids[0] - if len(subregion_ids) == 1: - subregion_id = cluster_id - name = get_osm_border_name_by_osm_id(g.conn, subregion_id) - else: - counter += 1 - free_id -= 1 - subregion_id = free_id - name = f"{base_name}_{counter}" - insert_cursor.execute(f""" - INSERT INTO {table} (id, name, parent_id, geom, modified, count_k, mwm_size_est) - SELECT {subregion_id}, %s, osm_border_id, geom, now(), -1, mwm_size_est - FROM {autosplit_table} WHERE subregion_ids[1] = %s AND {where_clause} - """, (name, cluster_id,) + splitting_sql_params - ) - g.conn.commit() - return jsonify(status='ok') @app.route('/chop1') def chop_largest_or_farthest(): @@ -791,6 +541,7 @@ def chop_largest_or_farthest(): g.conn.commit() return jsonify(status='ok', warnings=warnings) + @app.route('/hull') def draw_hull(): if config.READONLY: @@ -810,6 +561,7 @@ def draw_hull(): g.conn.commit() return jsonify(status='ok') + @app.route('/backup') def backup_do(): if config.READONLY: @@ -834,6 +586,7 @@ def backup_do(): g.conn.commit() return jsonify(status='ok') + @app.route('/restore') def backup_restore(): if config.READONLY: @@ -858,6 +611,7 @@ def backup_restore(): g.conn.commit() return jsonify(status='ok') + @app.route('/backlist') def backup_list(): cur = g.conn.cursor() @@ -871,6 +625,7 @@ def backup_list(): # todo: count number of different objects for the last one return jsonify(backups=result) + @app.route('/backdelete') def backup_delete(): if config.READONLY: @@ -885,6 +640,7 @@ def backup_delete(): g.conn.commit() return jsonify(status='ok') + @app.route('/josm') def make_osm(): xmin = request.args.get('xmin') @@ -904,6 +660,7 @@ def make_osm(): xml = borders_to_xml(borders) return Response(xml, mimetype='application/x-osm+xml') + @app.route('/josmbord') def josm_borders_along(): region_id = int(request.args.get('id')) @@ -965,7 +722,6 @@ def import_osm(): regions = result # submit modifications to the database - cur = g.conn.cursor() added = 0 updated = 0 free_id = None @@ -991,99 +747,6 @@ def import_osm(): g.conn.commit() return jsonify(regions=len(regions), added=added, updated=updated) -def get_free_id(): - cursor = g.conn.cursor() - table = config.TABLE - cursor.execute(f"SELECT min(id) FROM {table} WHERE id < -1000000000") - min_id = cursor.fetchone()[0] - free_id = min_id - 1 if min_id else -1_000_000_001 - return free_id - -def assign_region_to_lowerst_parent(region_id): - pot_parents = find_potential_parents(region_id) - if pot_parents: - # potential_parents are sorted by area ascending - parent_id = pot_parents[0]['properties']['id'] - cursor = g.conn.cursor() - table = config.TABLE - cursor.execute(f""" - UPDATE {table} - SET parent_id = %s - WHERE id = %s - """, (parent_id, region_id) - ) - return True - return False - -def create_or_update_region(region, free_id): - cursor = g.conn.cursor() - table = config.TABLE - if region['id'] < 0: - if not free_id: - free_id = get_free_id() - region_id = free_id - - cursor.execute(f""" - INSERT INTO {table} - (id, name, disabled, geom, modified, count_k) - VALUES (%s, %s, %s, ST_GeomFromText(%s, 4326), now(), -1) - """, (region_id, region['name'], region['disabled'], region['wkt']) - ) - assign_region_to_lowerst_parent(region_id) - return region_id - else: - cursor.execute(f"SELECT count(1) FROM {table} WHERE id = %s", - (-region['id'],)) - rec = cursor.fetchone() - if rec[0] == 0: - raise Exception(f"Can't find border ({region['id']}) for update") - cursor.execute(f""" - UPDATE {table} - SET disabled = %s, - name = %s, - modified = now(), - count_k = -1, - geom = ST_GeomFromText(%s, 4326) - WHERE id = %s - """, (region['disabled'], region['name'], - region['wkt'], -region['id']) - ) - return region['id'] - -def find_potential_parents(region_id): - table = config.TABLE - osm_table = config.OSM_TABLE - p_geogr = "geography(p.geom)" - c_geogr = "geography(c.geom)" - cursor = g.conn.cursor() - query = f""" - SELECT - p.id, - p.name, - (SELECT admin_level FROM {osm_table} WHERE osm_id = p.id) admin_level, - ST_AsGeoJSON(ST_SimplifyPreserveTopology(p.geom, 0.01)) geometry - FROM {table} p, {table} c - WHERE c.id = %s - AND ST_Intersects(p.geom, c.geom) - AND ST_Area({p_geogr}) > ST_Area({c_geogr}) - AND ST_Area(ST_Intersection({p_geogr}, {c_geogr})) > - 0.5 * ST_Area({c_geogr}) - ORDER BY ST_Area({p_geogr}) - """ - cursor.execute(query, (region_id,)) - parents = [] - for rec in cursor: - props = { - 'id': rec[0], - 'name': rec[1], - 'admin_level': rec[2], - } - feature = {'type': 'Feature', - 'geometry': json.loads(rec[3]), - 'properties': props - } - parents.append(feature) - return parents @app.route('/potential_parents') def potential_parents(): @@ -1091,6 +754,7 @@ def potential_parents(): parents = find_potential_parents(region_id) return jsonify(status='ok', parents=parents) + @app.route('/poly') def export_poly(): table = request.args.get('table') @@ -1147,6 +811,7 @@ def export_poly(): memory_file.seek(0) return send_file(memory_file, attachment_filename='borders.zip', as_attachment=True) + @app.route('/stat') def statistics(): group = request.args.get('group') @@ -1201,6 +866,7 @@ def statistics(): return jsonify(regions=result) return jsonify(status='wrong group id') + @app.route('/border') def border(): region_id = int(request.args.get('id')) @@ -1217,6 +883,7 @@ def border(): return jsonify(status=f'No border with id={region_id} found') return jsonify(status='ok', geojson=borders[0]) + @app.route('/start_over') def start_over(): try: diff --git a/web/app/borders_api_utils.py b/web/app/borders_api_utils.py new file mode 100644 index 0000000..387e320 --- /dev/null +++ b/web/app/borders_api_utils.py @@ -0,0 +1,373 @@ +import itertools +import json + +from flask import g, jsonify + +import config +from auto_split import split_region +from countries_structure import get_osm_border_name_by_osm_id +from subregions import ( + get_region_country, + get_subregions_info, + is_administrative_region, +) + + +def fetch_borders(**kwargs): + table = kwargs.get('table', config.TABLE) + simplify = kwargs.get('simplify', 0) + where_clause = kwargs.get('where_clause', '1=1') + only_leaves = kwargs.get('only_leaves', True) + osm_table = config.OSM_TABLE + geom = (f'ST_SimplifyPreserveTopology(geom, {simplify})' + if simplify > 0 else 'geom') + leaves_filter = (f""" AND id NOT IN (SELECT parent_id FROM {table} + WHERE parent_id IS NOT NULL)""" + if only_leaves else '') + query = f""" + SELECT name, geometry, nodes, modified, disabled, count_k, cmnt, + (CASE WHEN area = 'NaN'::DOUBLE PRECISION THEN 0 ELSE area END) AS area, + id, admin_level, parent_id, parent_name, parent_admin_level, + mwm_size_est + FROM ( + SELECT name, + ST_AsGeoJSON({geom}, 7) as geometry, + ST_NPoints(geom) AS nodes, + modified, + disabled, + count_k, + cmnt, + round(ST_Area(geography(geom))) AS area, + id, + ( SELECT admin_level FROM {osm_table} + WHERE osm_id = t.id + ) AS admin_level, + parent_id, + ( SELECT name FROM {table} + WHERE id = t.parent_id + ) AS parent_name, + ( SELECT admin_level FROM {osm_table} + WHERE osm_id = (SELECT parent_id FROM {table} WHERE id = t.id) + ) AS parent_admin_level, + mwm_size_est + FROM {table} t + WHERE ({where_clause}) {leaves_filter} + ) q + ORDER BY area DESC + """ + cur = g.conn.cursor() + cur.execute(query) + borders = [] + for rec in cur: + region_id = rec[8] + country_id, country_name = get_region_country(g.conn, region_id) + props = { 'name': rec[0] or '', 'nodes': rec[2], 'modified': rec[3], + 'disabled': rec[4], 'count_k': rec[5], + 'comment': rec[6], + 'area': rec[7], + 'id': region_id, + 'admin_level': rec[9], + 'parent_id': rec[10], + 'parent_name': rec[11], + 'parent_admin_level': rec[12], + 'country_id': country_id, + 'country_name': country_name, + 'mwm_size_est': rec[13] + } + feature = {'type': 'Feature', + 'geometry': json.loads(rec[1]), + 'properties': props + } + borders.append(feature) + return borders + + +def simplify_level_to_postgis_value(simplify_level): + return ( + 0.1 if simplify_level == '2' + else 0.01 if simplify_level == '1' + else 0 + ) + + +def get_subregions_for_preview(region_ids, next_level): + subregions = list(itertools.chain.from_iterable( + get_subregions_one_for_preview(region_id, next_level) + for region_id in region_ids + )) + return subregions + + +def get_subregions_one_for_preview(region_id, next_level): + osm_table = config.OSM_TABLE + table = config.TABLE + cur = g.conn.cursor() + # We use ST_SimplifyPreserveTopology, since ST_Simplify would give NULL + # for very little regions. + cur.execute(f""" + SELECT name, + ST_AsGeoJSON(ST_SimplifyPreserveTopology(way, 0.01)) as way, + osm_id + FROM {osm_table} + WHERE ST_Contains( + (SELECT geom FROM {table} WHERE id = %s), way + ) + AND admin_level = %s + """, (region_id, next_level) + ) + subregions = [] + for rec in cur: + feature = {'type': 'Feature', 'geometry': json.loads(rec[1]), + 'properties': {'name': rec[0]}} + subregions.append(feature) + return subregions + + +def get_clusters_for_preview(region_ids, next_level, thresholds): + clusters = list(itertools.chain.from_iterable( + get_clusters_for_preview_one(region_id, next_level, thresholds) + for region_id in region_ids + )) + return clusters + + +def get_clusters_for_preview_one(region_id, next_level, mwm_size_thr): + autosplit_table = config.AUTOSPLIT_TABLE + cursor = g.conn.cursor() + where_clause = f""" + osm_border_id = %s + AND mwm_size_thr = %s + """ + splitting_sql_params = (region_id, mwm_size_thr) + cursor.execute(f""" + SELECT 1 FROM {autosplit_table} + WHERE {where_clause} + """, splitting_sql_params + ) + if cursor.rowcount == 0: + split_region(g.conn, region_id, next_level, mwm_size_thr) + + cursor.execute(f""" + SELECT subregion_ids[1], + ST_AsGeoJSON(ST_SimplifyPreserveTopology(geom, 0.01)) as way + FROM {autosplit_table} + WHERE {where_clause} + """, splitting_sql_params + ) + clusters = [] + for rec in cursor: + cluster = { + 'type': 'Feature', + 'geometry': json.loads(rec[1]), + 'properties': {'osm_id': int(rec[0])} + } + clusters.append(cluster) + return clusters + + +def divide_into_subregions_preview(region_ids, next_level): + subregions = get_subregions_for_preview(region_ids, next_level) + return jsonify( + status='ok', + subregions={'type': 'FeatureCollection', 'features': subregions} + ) + + +def divide_into_clusters_preview(region_ids, next_level, mwm_size_thr): + subregions = get_subregions_for_preview(region_ids, next_level) + clusters = get_clusters_for_preview(region_ids, next_level, mwm_size_thr) + return jsonify( + status='ok', + subregions={'type': 'FeatureCollection', 'features': subregions}, + clusters={'type': 'FeatureCollection', 'features': clusters} + ) + + +def divide_into_subregions(region_ids, next_level): + for region_id in region_ids: + divide_into_subregions_one(region_id, next_level) + g.conn.commit() + return jsonify(status='ok') + + +def divide_into_subregions_one(region_id, next_level): + table = config.TABLE + osm_table = config.OSM_TABLE + subregions = get_subregions_info(g.conn, region_id, table, + next_level, need_cities=False) + cursor = g.conn.cursor() + is_admin_region = is_administrative_region(g.conn, region_id) + if is_admin_region: + for subregion_id, data in subregions.items(): + cursor.execute(f""" + INSERT INTO {table} + (id, geom, name, parent_id, modified, count_k, mwm_size_est) + SELECT osm_id, way, name, %s, now(), -1, {data['mwm_size_est']} + FROM {osm_table} + WHERE osm_id = %s + """, (region_id, subregion_id) + ) + else: + for subregion_id, data in subregions.items(): + cursor.execute(f""" + INSERT INTO {table} + (id, geom, name, parent_id, modified, count_k, mwm_size_est) + SELECT osm_id, way, name, + (SELECT parent_id FROM {table} WHERE id = %s), + now(), -1, {data['mwm_size_est']} + FROM {osm_table} + WHERE osm_id = %s + """, (region_id, subregion_id) + ) + cursor.execute(f"DELETE FROM {table} WHERE id = %s", (region_id,)) + + +def divide_into_clusters(region_ids, next_level, mwm_size_thr): + table = config.TABLE + autosplit_table = config.AUTOSPLIT_TABLE + cursor = g.conn.cursor() + insert_cursor = g.conn.cursor() + for region_id in region_ids: + cursor.execute(f"SELECT name FROM {table} WHERE id = %s", (region_id,)) + base_name = cursor.fetchone()[0] + + where_clause = f""" + osm_border_id = %s + AND mwm_size_thr = %s + """ + splitting_sql_params = (region_id, mwm_size_thr) + cursor.execute(f""" + SELECT 1 FROM {autosplit_table} + WHERE {where_clause} + """, splitting_sql_params + ) + if cursor.rowcount == 0: + split_region(g.conn, region_id, next_level, mwm_size_thr) + + free_id = get_free_id() + counter = 0 + cursor.execute(f""" + SELECT subregion_ids + FROM {autosplit_table} WHERE {where_clause} + """, splitting_sql_params + ) + if cursor.rowcount == 1: + continue + for rec in cursor: + subregion_ids = rec[0] + cluster_id = subregion_ids[0] + if len(subregion_ids) == 1: + subregion_id = cluster_id + name = get_osm_border_name_by_osm_id(g.conn, subregion_id) + else: + counter += 1 + free_id -= 1 + subregion_id = free_id + name = f"{base_name}_{counter}" + insert_cursor.execute(f""" + INSERT INTO {table} (id, name, parent_id, geom, modified, count_k, mwm_size_est) + SELECT {subregion_id}, %s, osm_border_id, geom, now(), -1, mwm_size_est + FROM {autosplit_table} WHERE subregion_ids[1] = %s AND {where_clause} + """, (name, cluster_id,) + splitting_sql_params + ) + g.conn.commit() + return jsonify(status='ok') + + +def get_free_id(): + cursor = g.conn.cursor() + table = config.TABLE + cursor.execute(f"SELECT min(id) FROM {table} WHERE id < -1000000000") + min_id = cursor.fetchone()[0] + free_id = min_id - 1 if min_id else -1_000_000_001 + return free_id + + +def assign_region_to_lowest_parent(region_id): + pot_parents = find_potential_parents(region_id) + if pot_parents: + # potential_parents are sorted by area ascending + parent_id = pot_parents[0]['properties']['id'] + cursor = g.conn.cursor() + table = config.TABLE + cursor.execute(f""" + UPDATE {table} + SET parent_id = %s + WHERE id = %s + """, (parent_id, region_id) + ) + return True + return False + + +def create_or_update_region(region, free_id): + cursor = g.conn.cursor() + table = config.TABLE + if region['id'] < 0: + if not free_id: + free_id = get_free_id() + region_id = free_id + + cursor.execute(f""" + INSERT INTO {table} + (id, name, disabled, geom, modified, count_k) + VALUES (%s, %s, %s, ST_GeomFromText(%s, 4326), now(), -1) + """, (region_id, region['name'], region['disabled'], region['wkt']) + ) + assign_region_to_lowest_parent(region_id) + return region_id + else: + cursor.execute(f"SELECT count(1) FROM {table} WHERE id = %s", + (-region['id'],)) + rec = cursor.fetchone() + if rec[0] == 0: + raise Exception(f"Can't find border ({region['id']}) for update") + cursor.execute(f""" + UPDATE {table} + SET disabled = %s, + name = %s, + modified = now(), + count_k = -1, + geom = ST_GeomFromText(%s, 4326) + WHERE id = %s + """, (region['disabled'], region['name'], + region['wkt'], -region['id']) + ) + return region['id'] + + +def find_potential_parents(region_id): + table = config.TABLE + osm_table = config.OSM_TABLE + p_geogr = "geography(p.geom)" + c_geogr = "geography(c.geom)" + cursor = g.conn.cursor() + query = f""" + SELECT + p.id, + p.name, + (SELECT admin_level FROM {osm_table} WHERE osm_id = p.id) admin_level, + ST_AsGeoJSON(ST_SimplifyPreserveTopology(p.geom, 0.01)) geometry + FROM {table} p, {table} c + WHERE c.id = %s + AND ST_Intersects(p.geom, c.geom) + AND ST_Area({p_geogr}) > ST_Area({c_geogr}) + AND ST_Area(ST_Intersection({p_geogr}, {c_geogr})) > + 0.5 * ST_Area({c_geogr}) + ORDER BY ST_Area({p_geogr}) + """ + cursor.execute(query, (region_id,)) + parents = [] + for rec in cursor: + props = { + 'id': rec[0], + 'name': rec[1], + 'admin_level': rec[2], + } + feature = { + 'type': 'Feature', + 'geometry': json.loads(rec[3]), + 'properties': props + } + parents.append(feature) + return parents diff --git a/web/app/countries_structure.py b/web/app/countries_structure.py index 4e828fc..cb1e1f0 100644 --- a/web/app/countries_structure.py +++ b/web/app/countries_structure.py @@ -1,5 +1,4 @@ import itertools -from queue import Queue from config import ( TABLE as table, @@ -385,104 +384,3 @@ def _get_country_osm_id_by_name(conn, name): if not rec: raise CountryStructureException(f'Not found country "{name}"') return int(rec[0]) - - -def is_administrative_region(conn, region_id): - cursor = conn.cursor() - cursor.execute(f""" - SELECT count(1) FROM {osm_table} WHERE osm_id = %s - """, (region_id,) - ) - count = cursor.fetchone()[0] - return (count > 0) - - -def find_osm_child_regions(conn, region_id): - cursor = conn.cursor() - cursor.execute(f""" - SELECT c.id, oc.admin_level - FROM {table} c, {table} p, {osm_table} oc - WHERE p.id = c.parent_id AND c.id = oc.osm_id - AND p.id = %s - """, (region_id,) - ) - children = [] - for rec in cursor: - children.append({'id': int(rec[0]), 'admin_level': int(rec[1])}) - return children - - -def is_leaf(conn, region_id): - cursor = conn.cursor() - cursor.execute(f""" - SELECT count(1) - FROM {table} - WHERE parent_id = %s - """, (region_id,) - ) - count = cursor.fetchone()[0] - return (count == 0) - - -def get_region_country(conn, region_id): - """Returns the uppermost predecessor of the region in the hierarchy, - possibly itself. - """ - predecessors = get_predecessors(conn, region_id) - return predecessors[-1] - - -def get_predecessors(conn, region_id): - """Returns the list of (id, name)-tuples of all predecessors, - starting from the very region_id. - """ - predecessors = [] - cursor = conn.cursor() - while True: - cursor.execute(f""" - SELECT id, name, parent_id - FROM {table} WHERE id={region_id} - """ - ) - rec = cursor.fetchone() - if not rec: - raise Exception(f"No record in '{table}' table with id = {region_id}") - predecessors.append(rec[0:2]) - parent_id = rec[2] - if not parent_id: - break - region_id = parent_id - return predecessors - - -def get_region_full_name(conn, region_id): - predecessors = get_predecessors(conn, region_id) - return '_'.join(pr[1] for pr in reversed(predecessors)) - - -def get_similar_regions(conn, region_id, only_leaves=False): - """Returns ids of regions of the same admin_level in the same country. - Prerequisite: is_administrative_region(region_id) is True. - """ - cursor = conn.cursor() - cursor.execute(f""" - SELECT admin_level FROM {osm_table} - WHERE osm_id = %s""", (region_id,) - ) - admin_level = int(cursor.fetchone()[0]) - country_id, country_name = get_region_country(conn, region_id) - q = Queue() - q.put({'id': country_id, 'admin_level': 2}) - similar_region_ids = [] - while not q.empty(): - item = q.get() - if item['admin_level'] == admin_level: - similar_region_ids.append(item['id']) - elif item['admin_level'] < admin_level: - children = find_osm_child_regions(item['id']) - for ch in children: - q.put(ch) - if only_leaves: - similar_region_ids = [r_id for r_id in similar_region_ids - if is_leaf(conn, r_id)] - return similar_region_ids diff --git a/web/app/subregions.py b/web/app/subregions.py index 9c5e97b..9a7e6c6 100644 --- a/web/app/subregions.py +++ b/web/app/subregions.py @@ -1,9 +1,11 @@ import math +from queue import Queue import config from mwm_size_predictor import MwmSizePredictor +table = config.TABLE osm_table = config.OSM_TABLE osm_places_table = config.OSM_PLACES_TABLE @@ -33,7 +35,7 @@ def _get_subregions_basic_info(conn, region_id, region_table, next_level, need_cities): cursor = conn.cursor() region_id_column, region_geom_column = ( - ('id', 'geom') if region_table == config.TABLE else + ('id', 'geom') if region_table == table else ('osm_id', 'way') ) cursor.execute(f""" @@ -106,7 +108,6 @@ def _add_mwm_size_estimation(subregions): def update_border_mwm_size_estimation(conn, border_id): - table = config.TABLE cursor = conn.cursor() cursor.execute(f""" SELECT name, ST_Area(geography(geom))/1.0E+6 area @@ -142,3 +143,125 @@ def update_border_mwm_size_estimation(conn, border_id): cursor.execute(f"UPDATE {table} SET mwm_size_est = %s WHERE id = %s", (mwm_size_est, border_id)) conn.commit() + + +def is_administrative_region(conn, region_id): + cursor = conn.cursor() + cursor.execute(f""" + SELECT count(1) FROM {osm_table} WHERE osm_id = %s + """, (region_id,) + ) + count = cursor.fetchone()[0] + return (count > 0) + + +def is_leaf(conn, region_id): + cursor = conn.cursor() + cursor.execute(f""" + SELECT count(1) + FROM {table} + WHERE parent_id = %s + """, (region_id,) + ) + count = cursor.fetchone()[0] + return (count == 0) + + +def get_region_country(conn, region_id): + """Returns the uppermost predecessor of the region in the hierarchy, + possibly itself. + """ + predecessors = get_predecessors(conn, region_id) + return predecessors[-1] + + +def get_predecessors(conn, region_id): + """Returns the list of (id, name)-tuples of all predecessors, + starting from the very region_id. + """ + predecessors = [] + cursor = conn.cursor() + while True: + cursor.execute(f""" + SELECT id, name, parent_id + FROM {table} WHERE id = %s + """, (region_id,) + ) + rec = cursor.fetchone() + if not rec: + raise Exception(f"No record in '{table}' table with id = {region_id}") + predecessors.append(rec[0:2]) + parent_id = rec[2] + if not parent_id: + break + region_id = parent_id + return predecessors + + +def get_region_full_name(conn, region_id): + predecessors = get_predecessors(conn, region_id) + return '_'.join(pr[1] for pr in reversed(predecessors)) + + +def get_parent_region_id(conn, region_id): + cursor = conn.cursor() + cursor.execute(f""" + SELECT parent_id FROM {table} WHERE id = %s + """, (region_id,)) + rec = cursor.fetchone() + parent_id = int(rec[0]) if rec and rec[0] is not None else None + return parent_id + + +def get_child_region_ids(conn, region_id): + cursor = conn.cursor() + cursor.execute(f""" + SELECT id FROM {table} WHERE parent_id = %s + """, (region_id,)) + child_ids = [] + for rec in cursor: + child_ids.append(int(rec[0])) + return child_ids + + +def get_similar_regions(conn, region_id, only_leaves=False): + """Returns ids of regions of the same admin_level in the same country. + Prerequisite: is_administrative_region(region_id) is True. + """ + cursor = conn.cursor() + cursor.execute(f""" + SELECT admin_level FROM {osm_table} + WHERE osm_id = %s""", (region_id,) + ) + admin_level = int(cursor.fetchone()[0]) + country_id, country_name = get_region_country(conn, region_id) + q = Queue() + q.put({'id': country_id, 'admin_level': 2}) + similar_region_ids = [] + while not q.empty(): + item = q.get() + if item['admin_level'] == admin_level: + similar_region_ids.append(item['id']) + elif item['admin_level'] < admin_level: + children = find_osm_child_regions(item['id']) + for ch in children: + q.put(ch) + if only_leaves: + similar_region_ids = [r_id for r_id in similar_region_ids + if is_leaf(conn, r_id)] + return similar_region_ids + + +def find_osm_child_regions(conn, region_id): + cursor = conn.cursor() + cursor.execute(f""" + SELECT c.id, oc.admin_level + FROM {table} c, {table} p, {osm_table} oc + WHERE p.id = c.parent_id AND c.id = oc.osm_id + AND p.id = %s + """, (region_id,) + ) + children = [] + for rec in cursor: + children.append({'id': int(rec[0]), 'admin_level': int(rec[1])}) + return children