From d46dca1df1b83450d827963a41e84cadc0fcaa1f Mon Sep 17 00:00:00 2001
From: Alexey Zakharenkov
<35913079+alexey-zakharenkov@users.noreply.github.com>
Date: Fri, 16 Oct 2020 11:03:54 +0300
Subject: [PATCH] Repair 'Extract islands' operation; add automatic
recalculation of mwm size estimation after some operations
---
web/app/borders_api.py | 47 +++++++++++++++++++++++++----------
web/app/mwm_size_predictor.py | 15 ++++++++---
web/app/static/borders.js | 4 +--
web/app/subregions.py | 40 +++++++++++++++++++++++++++--
web/app/templates/index.html | 2 +-
5 files changed, 87 insertions(+), 21 deletions(-)
diff --git a/web/app/borders_api.py b/web/app/borders_api.py
index 251dc1e..082e1e3 100755
--- a/web/app/borders_api.py
+++ b/web/app/borders_api.py
@@ -25,7 +25,11 @@ from countries_structure import (
create_countries_initial_structure,
get_osm_border_name_by_osm_id,
)
-from subregions import get_subregions_info
+from subregions import (
+ get_subregions_info,
+ update_border_mwm_size_estimation,
+)
+
try:
from lxml import etree
@@ -279,6 +283,7 @@ def split():
base_name = name
# insert new geometries
counter = 1
+ new_ids = []
free_id = get_free_id()
for geom in geometries:
cur.execute(f"""
@@ -286,8 +291,11 @@ def split():
VALUES (%s, %s, ST_GeomFromText(%s, 4326), %s, -1, now(), %s)
""", (free_id, f'{base_name}_{counter}', geom, disabled, parent_id)
)
+ new_ids.append(free_id)
counter += 1
free_id -= 1
+ for border_id in new_ids:
+ update_border_mwm_size_estimation(g.conn, border_id)
g.conn.commit()
return jsonify(status='ok')
@@ -430,6 +438,7 @@ def copy_from_osm():
""", (osm_id,)
)
assign_region_to_lowerst_parent(osm_id)
+ update_border_mwm_size_estimation(g.conn, osm_id)
g.conn.commit()
return jsonify(status='ok')
@@ -818,23 +827,35 @@ def divide_into_clusters(region_ids, next_level, mwm_size_thr):
def chop_largest_or_farthest():
if config.READONLY:
abort(405)
- name = request.args.get('name').encode('utf-8')
+ region_id = int(request.args.get('id'))
+ table = config.TABLE
cur = g.conn.cursor()
- cur.execute('select ST_NumGeometries(geom) from {} where name = %s;'.format(config.TABLE), (name,))
+ cur.execute(f"""SELECT ST_NumGeometries(geom)
+ FROM {table}
+ WHERE id = {region_id}""")
res = cur.fetchone()
if not res or res[0] < 2:
return jsonify(status='border should have more than one outer ring')
- cur.execute("""INSERT INTO {table} (name, disabled, modified, geom)
- SELECT name, disabled, modified, geom from
+ free_id1 = get_free_id()
+ free_id2 = free_id1 - 1
+ cur.execute(f"""
+ INSERT INTO {table} (id, parent_id, name, disabled, modified, geom)
+ SELECT id, region_id, name, disabled, modified, geom FROM
(
- (WITH w AS (SELECT name, disabled, (ST_Dump(geom)).geom AS g FROM {table} WHERE name = %s)
- (SELECT name||'_main' as name, disabled, now() as modified, g as geom, ST_Area(g) as a FROM w ORDER BY a DESC LIMIT 1)
- UNION ALL
- SELECT name||'_small' as name, disabled, now() as modified, ST_Collect(g) AS geom, ST_Area(ST_Collect(g)) as a
- FROM (SELECT name, disabled, g, ST_Area(g) AS a FROM w ORDER BY a DESC OFFSET 1) ww
- GROUP BY name, disabled)
- ) x;""".format(table=config.TABLE), (name,))
- cur.execute('delete from {} where name = %s;'.format(config.TABLE), (name,))
+ (WITH w AS (SELECT name, disabled, (ST_Dump(geom)).geom AS g
+ FROM {table} WHERE id = {region_id})
+ (SELECT {free_id1} id, {region_id} region_id, name||'_main' as name, disabled,
+ now() as modified, g as geom, ST_Area(g) as a
+ FROM w ORDER BY a DESC LIMIT 1)
+ UNION ALL
+ SELECT {free_id2} id, {region_id} region_id, name||'_small' as name, disabled,
+ now() as modified, ST_Collect(g) AS geom,
+ ST_Area(ST_Collect(g)) as a
+ FROM (SELECT name, disabled, g, ST_Area(g) AS a FROM w ORDER BY a DESC OFFSET 1) ww
+ GROUP BY name, disabled)
+ ) x""")
+ for border_id in (free_id1, free_id2):
+ update_border_mwm_size_estimation(g.conn, border_id)
g.conn.commit()
return jsonify(status='ok')
diff --git a/web/app/mwm_size_predictor.py b/web/app/mwm_size_predictor.py
index 112ff78..4045635 100644
--- a/web/app/mwm_size_predictor.py
+++ b/web/app/mwm_size_predictor.py
@@ -12,7 +12,14 @@ class MwmSizePredictor:
with open(config.MWM_SIZE_PREDICTION_MODEL_SCALER_PATH, 'rb') as f:
self.scaler = pickle.load(f)
- def predict(self, features_array):
+ @classmethod
+ def _get_instance(cls):
+ if not hasattr(cls, '_instance'):
+ cls._instance = cls()
+ return cls._instance
+
+ @classmethod
+ def predict(cls, features_array):
"""1D or 2D array of feature values for predictions. Features are
'urban_pop', 'area', 'city_cnt', 'hamlet_cnt' as defined for the
prediction model.
@@ -21,8 +28,10 @@ class MwmSizePredictor:
one_prediction = (X.ndim == 1)
if one_prediction:
X = X.reshape(1, -1)
- X_scaled = self.scaler.transform(X)
- predictions = self.model.predict(X_scaled)
+
+ predictor = cls._get_instance()
+ X_scaled = predictor.scaler.transform(X)
+ predictions = predictor.model.predict(X_scaled)
if one_prediction:
return predictions[0]
else:
diff --git a/web/app/static/borders.js b/web/app/static/borders.js
index 2ac3a54..351793c 100644
--- a/web/app/static/borders.js
+++ b/web/app/static/borders.js
@@ -1040,8 +1040,8 @@ function bLargest() {
if( !selectedId || !(selectedId in borders) )
return;
$.ajax(getServer('chop1'), {
- data: { 'name': selectedId },
- success: updateBorders
+ data: { 'id': selectedId },
+ success: makeAnswerHandler(updateBorders)
});
}
diff --git a/web/app/subregions.py b/web/app/subregions.py
index d5ffff4..3dd0dc9 100644
--- a/web/app/subregions.py
+++ b/web/app/subregions.py
@@ -4,7 +4,6 @@ from mwm_size_predictor import MwmSizePredictor
osm_table = config.OSM_TABLE
osm_places_table = config.OSM_PLACES_TABLE
-size_predictor = MwmSizePredictor()
def get_subregions_info(conn, region_id, region_table,
@@ -93,10 +92,47 @@ def _add_mwm_size_estimation(subregions):
]
feature_array = [x[1] for x in subregions_sorted]
- predictions = size_predictor.predict(feature_array)
+ predictions = MwmSizePredictor.predict(feature_array)
for subregion_id, mwm_size_prediction in zip(
(x[0] for x in subregions_sorted),
predictions
):
subregions[subregion_id]['mwm_size_est'] = mwm_size_prediction
+
+
+def update_border_mwm_size_estimation(conn, border_id):
+ table = config.TABLE
+ cursor = conn.cursor()
+ cursor.execute(f"""
+ SELECT ST_Area(geography(geom))/1.0E+6 area
+ FROM {table}
+ WHERE id = %s""", (border_id, ))
+ rec = cursor.fetchone()
+ border_data = {
+ 'area': rec[0],
+ 'urban_pop': 0,
+ 'city_cnt': 0,
+ 'hamlet_cnt': 0
+ }
+ cursor.execute(f"""
+ SELECT COALESCE(p.population, 0), p.place
+ FROM {table} b, {config.OSM_PLACES_TABLE} p
+ WHERE id = %s
+ AND ST_CONTAINS(b.geom, p.center)
+ """, (border_id, ))
+ for place_population, place_type in cursor:
+ if place_type in ('city', 'town'):
+ border_data['city_cnt'] += 1
+ border_data['urban_pop'] += place_population
+ else:
+ border_data['hamlet_cnt'] += 1
+
+ feature_array = [
+ border_data[f] for f in
+ ('urban_pop', 'area', 'city_cnt', 'hamlet_cnt')
+ ]
+ mwm_size_est = MwmSizePredictor.predict(feature_array)
+ cursor.execute(f"UPDATE {table} SET mwm_size_est = %s WHERE id = %s",
+ (mwm_size_est, border_id))
+ conn.commit()
diff --git a/web/app/templates/index.html b/web/app/templates/index.html
index 8ce9d4d..6c3e558 100644
--- a/web/app/templates/index.html
+++ b/web/app/templates/index.html
@@ -121,7 +121,7 @@
-
+