old borders layer, statistics

2015-03-25 16:40:54 +03:00 · 2015-03-25 16:40:54 +03:00 · e46fc05076
commit e46fc05076
parent 83f0f6cfe8
5 changed files with 349 additions and 26 deletions
--- a/server/borders-api.py
+++ b/server/borders-api.py
@ -8,9 +8,10 @@ import psycopg2

 TABLE = 'borders'
 OSM_TABLE = 'osm_borders'
+OTHER_TABLES = { 'old': 'old_borders' }
 BACKUP = 'borders_backup'
 READONLY = False
-
+SMALL_KM2 = 10
 JOSM_FORCE_MULTI = True

 app = Flask(__name__)
@ -45,13 +46,20 @@ def query_bbox():
 		simplify = 0.01
 	else:
 		simplify = 0
+	table = request.args.get('table')
+	if table in OTHER_TABLES:
+		table = OTHER_TABLES[table]
+	else:
+		table = TABLE
+
 	cur = g.conn.cursor()
-	cur.execute('''SELECT name, ST_AsGeoJSON({geom}, 7) as geometry, ST_NPoints(geom),
-		modified, disabled, count_k, cmnt, round(ST_Area(geography(geom))) as area
+	cur.execute("""SELECT name, ST_AsGeoJSON({geom}, 7) as geometry, ST_NPoints(geom),
+		modified, disabled, count_k, cmnt,
+		round(CASE WHEN ST_Area(geography(geom)) = 'NaN' THEN 0 ELSE ST_Area(geography(geom)) END) as area
 		FROM {table}
 		WHERE geom && ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s, %s))
 		order by area desc;
-		'''.format(table=TABLE, geom='ST_SimplifyPreserveTopology(geom, {})'.format(simplify) if simplify > 0 else 'geom'),
+		""".format(table=table, geom='ST_SimplifyPreserveTopology(geom, {})'.format(simplify) if simplify > 0 else 'geom'),
 		(xmin, ymin, xmax, ymax))
 	result = []
 	for rec in cur:
@ -66,6 +74,11 @@ def query_small_in_bbox():
 	xmax = request.args.get('xmax')
 	ymin = request.args.get('ymin')
 	ymax = request.args.get('ymax')
+	table = request.args.get('table')
+	if table in OTHER_TABLES:
+		table = OTHER_TABLES[table]
+	else:
+		table = TABLE
 	cur = g.conn.cursor()
 	cur.execute('''SELECT name, round(ST_Area(geography(ring))) as area, ST_X(ST_Centroid(ring)), ST_Y(ST_Centroid(ring))
 		FROM (
@ -73,23 +86,35 @@ def query_small_in_bbox():
 			FROM {table}
 			WHERE geom && ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s, %s))
 		) g
-		WHERE ST_Area(geography(ring)) < 1000000;'''.format(table=TABLE), (xmin, ymin, xmax, ymax))
+		WHERE ST_Area(geography(ring)) < %s;'''.format(table=table), (xmin, ymin, xmax, ymax, SMALL_KM2 * 1000000))
 	result = []
 	for rec in cur:
 		result.append({ 'name': rec[0], 'area': rec[1], 'lon': float(rec[2]), 'lat': float(rec[3]) })
 	return jsonify(features=result)

-@app.route('/hasosm')
+@app.route('/tables')
 def check_osm_table():
-	res = False
+	table = request.args.get('table')
+	if table in OTHER_TABLES:
+		table = OTHER_TABLES[table]
+	else:
+		table = TABLE
+	osm = False
+	old = False
 	try:
 		cur = g.conn.cursor()
 		cur.execute('select osm_id, ST_Area(way), admin_level, name from {} limit 2;'.format(OSM_TABLE))
 		if cur.rowcount == 2:
-			res = True
+			osm = True
 	except psycopg2.Error, e:
 		pass
-	return jsonify(result=res)
+	try:
+		cur.execute('select name, ST_Area(geom), modified, disabled, count_k, cmnt from {} limit 2;'.format(table))
+		if cur.rowcount == 2:
+			old = True
+	except psycopg2.Error, e:
+		pass
+	return jsonify(osm=osm, table=old)

@app.route('/split')
 def split():
@ -327,8 +352,14 @@ def make_osm():
 	xmax = request.args.get('xmax')
 	ymin = request.args.get('ymin')
 	ymax = request.args.get('ymax')
+	table = request.args.get('table')
+	if table in OTHER_TABLES:
+		table = OTHER_TABLES[table]
+	else:
+		table = TABLE
+
 	cur = g.conn.cursor()
-	cur.execute('SELECT name, disabled, ST_AsGeoJSON(geom, 7) as geometry FROM {table} WHERE ST_Intersects(ST_SetSRID(ST_Buffer(ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s, %s)), 0.3), 4326), geom);'.format(table=TABLE), (xmin, ymin, xmax, ymax))
+	cur.execute('SELECT name, disabled, ST_AsGeoJSON(geom, 7) as geometry FROM {table} WHERE ST_Intersects(ST_SetSRID(ST_Buffer(ST_MakeBox2D(ST_Point(%s, %s), ST_Point(%s, %s)), 0.3), 4326), geom);'.format(table=table), (xmin, ymin, xmax, ymax))

 	node_pool = { 'id': 1 } # 'lat_lon': id
 	regions = [] # { name: name, rings: [['outer', [ids]], ['inner', [ids]], ...] }
@ -534,7 +565,8 @@ def import_osm():
 		if rel.get('action') == 'delete':
 			continue
 		if len(outer) == 0:
-			return import_error('relation {} has no outer ways'.format(rel.get('id')))
+			continue
+			#return import_error('relation {} has no outer ways'.format(rel.get('id')))
 		# reconstruct rings in multipolygon
 		for multi in (inner, outer):
 			i = 0
@ -559,6 +591,11 @@ def import_osm():
 					if not productive:
 						return import_error('unconnected way in relation {}'.format(rel.get('id')))
 				i = i + 1
+		# check for 2-node rings
+		for multi in (outer, inner):
+			for way in multi:
+				if len(way['nodes']) < 3:
+					return import_error('Way in relation {} has only {} nodes'.format(rel.get('id'), len(way['nodes'])))
 		# sort inner and outer rings
 		polygons = []
 		for way in outer:
@ -577,7 +614,9 @@ def import_osm():
 		if not w['name']:
 			return import_error('unused in multipolygon way with no name: {}'.format(wid))
 		if w['nodes'][0] != w['nodes'][-1]:
-			return import_error('non-closed unused in multipolygon way: {}'.format(way.get('id')))
+			return import_error('non-closed unused in multipolygon way: {}'.format(wid))
+		if len(w['nodes']) < 3:
+			return import_error('way {} has {} nodes'.format(wid, len(w['nodes'])))
 		if w['name'] in regions:
 			return import_error('way {} has the same name as other way/multipolygon'.format(wid))
 		regions[w['name']] = { 'modified': w['modified'], 'disabled': w['disabled'], 'wkt': 'POLYGON({})'.format(way_to_wkt(nodes, w['nodes'])) }
@ -591,16 +630,43 @@ def import_osm():
 			continue
 		cur.execute('select count(1) from {} where name = %s'.format(TABLE), (name,))
 		res = cur.fetchone()
-		if res and res[0] > 0:
-			# update
-			cur.execute('update {table} set disabled = %s, geom = ST_GeomFromText(%s, 4326), modified = now(), count_k = -1 where name = %s'.format(table=TABLE), (region['disabled'], region['wkt'], name))
-			updated = updated + 1
-		else:
-			# create
-			cur.execute('insert into {table} (name, disabled, geom, modified, count_k) values (%s, %s, ST_GeomFromText(%s, 4326), now(), -1);'.format(table=TABLE), (name, region['disabled'], region['wkt']))
-			added = added + 1
+		try:
+			if res and res[0] > 0:
+				# update
+				cur.execute('update {table} set disabled = %s, geom = ST_GeomFromText(%s, 4326), modified = now(), count_k = -1 where name = %s'.format(table=TABLE), (region['disabled'], region['wkt'], name))
+				updated = updated + 1
+			else:
+				# create
+				cur.execute('insert into {table} (name, disabled, geom, modified, count_k) values (%s, %s, ST_GeomFromText(%s, 4326), now(), -1);'.format(table=TABLE), (name, region['disabled'], region['wkt']))
+				added = added + 1
+		except psycopg2.Error, e:
+			print 'WKT: {}'.format(region['wkt'])
+			raise
 	g.conn.commit()
 	return jsonify(regions=len(regions), added=added, updated=updated)

+@app.route('/stat')
+def statistics():
+	group = request.args.get('group')
+	cur = g.conn.cursor()
+	if group == 'total':
+		cur.execute('select count(1) from borders;')
+		return jsonify(total=cur.fetchone()[0])
+	elif group == 'sizes':
+		cur.execute("select name, count_k, ST_NPoints(geom), ST_AsGeoJSON(ST_Centroid(geom)), (case when ST_Area(geography(geom)) = 'NaN' then 0 else ST_Area(geography(geom)) / 1000000 end) as area from borders;")
+		result = []
+		for res in cur:
+			coord = json.loads(res[3])['coordinates']
+			result.append({ 'name': res[0], 'lat': coord[1], 'lon': coord[0], 'size': res[1], 'nodes': res[2], 'area': res[4] })
+		return jsonify(regions=result)
+	elif group == 'topo':
+		cur.execute("select name, count(1), min(case when ST_Area(geography(g)) = 'NaN' then 0 else ST_Area(geography(g)) end) / 1000000, sum(ST_NumInteriorRings(g)), ST_AsGeoJSON(ST_Centroid(ST_Collect(g))) from (select name, (ST_Dump(geom)).geom as g from borders) a group by name;")
+		result = []
+		for res in cur:
+			coord = json.loads(res[4])['coordinates']
+			result.append({ 'name': res[0], 'outer': res[1], 'min_area': res[2], 'inner': res[3], 'lon': coord[0], 'lat': coord[1] })
+		return jsonify(regions=result)
+	return jsonify(status='wrong group id')
+
 if __name__ == '__main__':
 	app.run(threaded=True)
--- a/www/borders.js
+++ b/www/borders.js
@ -3,12 +3,12 @@ var STYLE_SELECTED = { stroke: true, color: '#ff3', weight: 3, fill: true, fillO
 var FILL_TOO_SMALL = '#0f0';
 var FILL_TOO_BIG = '#800';
 var FILL_ZERO = 'black';
-var MB_TOO_BIG = 100;
-var KM2_AREA_TOO_SMALL = 1;
+var OLD_BORDERS_NAME = 'old';

 var map, borders = {}, bordersLayer, selectedId, editing = false;
 var size_good = 5, size_bad = 100;
 var tooSmallLayer = null;
+var oldBordersLayer = null;

 function init() {
 	map = L.map('map', { editable: true }).setView([30, 0], 3);
@ -31,8 +31,16 @@ function init() {
 }

 function checkHasOSM() {
-	$.ajax(server + '/hasosm', {
-		success: function(res) { if( res.result ) $('#osm_actions').css('display', 'block'); }
+	$.ajax(server + '/tables', {
+		data: { 'table': OLD_BORDERS_NAME },
+		success: function(res) {
+			if( res.osm )
+				$('#osm_actions').css('display', 'block');
+			if( res.table ) {
+				$('#old_action').css('display', 'block');
+				$('#josm_old').css('display', 'inline');
+			}
+		}
 	});
 }

@ -51,6 +59,22 @@ function updateBorders() {
 		dataType: 'json',
 		simplified: simplified
 	});
+
+	if( oldBordersLayer != null ) {
+		oldBordersLayer.clearLayers();
+		$.ajax(server + '/bbox', {
+			data: {
+				'table': OLD_BORDERS_NAME,
+				'simplify': simplified,
+				'xmin': b.getWest(),
+				'xmax': b.getEast(),
+				'ymin': b.getSouth(),
+				'ymax': b.getNorth()
+			},
+			success: processOldBorders,
+			dataType: 'json'
+		});
+	}
 }

 function processResult(data) {
@ -74,9 +98,9 @@ function processResult(data) {
 		selectLayer(null);
 	}

+	var b = map.getBounds();
 	if( tooSmallLayer != null ) {
 		tooSmallLayer.clearLayers();
-		var b = map.getBounds();
 		$.ajax(server + '/small', {
 			data: {
 				'xmin': b.getWest(),
@ -90,6 +114,13 @@ function processResult(data) {
 	}
 }

+function processOldBorders(data) {
+	var layer = L.geoJson(data, {
+		style: { fill: false, color: 'purple', weight: 3, clickable: false }
+	});
+	oldBordersLayer.addLayer(layer);
+}
+
 function processTooSmall(data) {
 	if( tooSmallLayer == null || !data || !('features' in data) )
 		return;
@ -232,6 +263,17 @@ function bUpdateColors() {
 	updateBorders();
 }

+function bOldBorders() {
+	if( $('#old').prop('checked') ) {
+		oldBordersLayer = L.layerGroup();
+		map.addLayer(oldBordersLayer);
+		updateBorders();
+	} else if( oldBordersLayer != null ) {
+		map.removeLayer(oldBordersLayer);
+		oldBordersLayer = null;
+	}
+}
+
 function bJOSM() {
 	var b = map.getBounds();
 	var url = server + '/josm?' + $.param({
@ -250,6 +292,25 @@ function bJOSM() {
 	});
 }

+function bJosmOld() {
+	var b = map.getBounds();
+	var url = server + '/josm?' + $.param({
+		'table': OLD_BORDERS_NAME,
+		'xmin': b.getWest(),
+		'xmax': b.getEast(),
+		'ymin': b.getSouth(),
+		'ymax': b.getNorth()
+	});
+	$.ajax({
+		url: 'http://127.0.0.1:8111/import',
+		data: { url: url, new_layer: 'true' },
+		complete: function(t) {
+			if( t.status != 200 )
+				window.alert('Please enable remote_control in JOSM');
+		}
+	});
+}
+
 function bJosmZoom() {
 	var b = map.getBounds();
 	$.ajax({
--- a/www/index.html
+++ b/www/index.html
@ -24,6 +24,7 @@
 			#header { border-bottom: 1px solid gray; margin-bottom: 1em; padding-bottom: 1em; }
 			#f_topo, #f_chars, #f_comments { font-size: 10pt; }
 			#backup_saving, #backup_restoring { margin-bottom: 1em; }
+			#old_action, #josm_old { display: none; }
 		</style>
 	</head>
 	<body onload="init();">
@ -53,13 +54,17 @@
 				</div>
 				<div id="b_josm">
 					<button onclick="bJOSM()">Открыть в JOSM</button>
+					<button id="josm_old" onclick="bJosmOld()">ст.</button>
 					<button onclick="bJosmZoom()">&#x1f50d;</button>
 				</div>
 				<form action="" enctype="multipart/form-data" method="post" id="filefm" target="import_frame">
 					Импорт <input type="file" accept=".osm,.xml" name="file" id="b_import" onchange="bImport();" style="max-width: 100px;">
 				</form>
 				<iframe name="import_frame" style="display: none;" width="230" height="80" src="about:blank"></iframe>
-				<button onclick="bBackup()">Архив границ</button>
+				<button onclick="bBackup()">Архив границ</button><br>
+				<div id="old_action">
+					<input type="checkbox" id="old" onchange="bOldBorders()"><label for="old"> старые границы</label>
+				</div>
 			</div>
 			<div id="actions" class="actions">
 				<button onclick="bDisable()" id="b_disable">Убрать</button>
--- a/www/stat.html
+++ b/www/stat.html
@ -0,0 +1,53 @@
+<!doctype html>
+<html>
+	<head>
+		<meta charset="utf-8">
+		<title>Статистика границ для MAPS.ME</title>
+		<script src="lib/jquery-1.11.2.min.js"></script>
+		<script src="stat.js"></script>
+		<script>server = 'http://127.0.0.1:5000';</script>
+		<style>
+			body > div { display: none; margin-bottom: 1em; }
+			#sizes > div { margin-top: 1em; }
+			.h { display: none; padding-left: 1em; }
+		</style>
+	</head>
+	<body onload="statInit();">
+		<h1>Статистика по границам</h1>
+		<div id="total">
+			Всего границ: <span id="total_total"></span><br>
+		</div>
+		<div id="sizes">
+			<div>
+				Названий с пробелами: <span id="names_spaces"></span><br>
+				Названий с левыми символами: <span id="names_bad"></span> (<a href="#" onclick="return statOpen('names_bad_list');">список</a>)<br>
+				<div id="names_bad_list" class="h"></div>
+			</div>
+			<div>
+				Размер MWM до 1 МБ: <span id="sizes_1mb"></span> (<a href="#" onclick="return statOpen('sizes_1mb_list');">список</a>)<br>
+				<div id="sizes_1mb_list" class="h"></div>
+				Размер MWM больше 50 МБ: <span id="sizes_50mb"></span> (<a href="#" onclick="return statOpen('sizes_50mb_list');">список</a>)<br>
+				<div id="sizes_50mb_list" class="h"></div>
+				Из них больше 100 МБ: <span id="sizes_100mb"></span> (<a href="#" onclick="return statOpen('sizes_100mb_list');">список</a>)<br>
+				<div id="sizes_100mb_list" class="h"></div>
+			</div>
+			<div>
+				Регионов меньше 100 км²: <span id="areas_100km"></span> (<a href="#" onclick="return statOpen('areas_100km_list');">список</a>)<br>
+				<div id="areas_100km_list" class="h"></div>
+				Регионов с 50k+ точек в контуре: <span id="areas_50k_points"></span> (<a href="#" onclick="return statOpen('areas_50k_points_list');">список</a>)<br>
+				<div id="areas_50k_points_list" class="h"></div>
+				Регионов с неизвестной площадью: <span id="areas_0"></span> (<a href="#" onclick="return statOpen('areas_0_list');">список</a>)<br>
+				<div id="areas_0_list" class="h"></div>
+			</div>
+		</div>
+		<div id="topo">
+			Регионов с дырками: <span id="topo_holes"></span> (<a href="#" onclick="return statOpen('topo_holes_list');">список</a>)<br>
+			<div id="topo_holes_list" class="h"></div>
+			Регионов из нескольких частей: <span id="topo_multi"></span> (<a href="#" onclick="return statOpen('topo_multi_list');">список</a>)<br>
+			<div id="topo_multi_list" class="h"></div>
+			Регионов с островами меньше 100 км²: <span id="topo_100km"></span> (<a href="#" onclick="return statOpen('topo_100km_list');">список</a>)<br>
+			<div id="topo_100km_list" class="h"></div>
+			<hr>
+		</div>
+	</body>
+</html>
--- a/www/stat.js
+++ b/www/stat.js
@ -0,0 +1,138 @@
+function statInit() {
+	statQuery('total', statTotal);
+}
+
+function statOpen(id) {
+	var div = document.getElementById(id);
+	if( div.style.display != 'block' )
+		div.style.display = 'block';
+	else
+		div.style.display = 'none';
+}
+
+function statQuery(id, callback) {
+	$.ajax(server + '/stat', {
+		data: { 'group': id },
+		success: function(data) {
+			callback(data);
+			document.getElementById(id).style.display = 'block';
+		},
+		error: function() { alert('Failed!'); }
+	});
+
+}
+
+function formatNum(value, digits) {
+	if( digits != undefined ) {
+		var pow = Math.pow(10, digits);
+		return Math.round(value * pow) / pow;
+	} else
+		return value;
+}
+
+function statFill(id, value, digits) {
+	document.getElementById(id).innerHTML = ('' + formatNum(value, digits)).replace('&', '&amp;').replace('<', '&lt;');
+}
+
+function getIndexLink(region) {
+	var big = region.area > 1000;
+	return 'index.html#' + (big ? 7 : 12) + '/' + region.lat + '/' + region.lon;
+}
+
+function statFillList(id, regions, comment, count) {
+	var div = document.getElementById(id), i, a, html, p;
+	if( !div ) {
+		console.log('Div ' + id + ' not found');
+		return;
+	}
+	if( count )
+		statFill(count, regions.length);
+	for( i = 0; i < regions.length; i++ ) {
+		a = document.createElement('a');
+		a.href = getIndexLink(regions[i]);
+		a.target = '_blank';
+		html = regions[i].name;
+		if( comment ) {
+			if( typeof comment == 'string' )
+				p = regions[i][comment];
+			else
+				p = comment(regions[i]);
+			if( p )
+				html += ' (' + p + ')';
+		}
+		a.innerHTML = html.replace('&', '&amp;').replace('<', '&lt;');
+		div.appendChild(a);
+		div.appendChild(document.createElement('br'));
+	}
+}
+
+function statTotal(data) {
+	statFill('total_total', data.total);
+	statQuery('sizes', statSizes);
+}
+
+function statSizes(data) {
+	var list_1mb = [], list_50mb = [], list_100mb = [];
+	var list_spaces = [], list_bad = [];
+	var list_100km = [], list_100kp = [], list_zero = [];
+
+	for( var i = 0; i < data.regions.length; i++ ) {
+		region = data.regions[i];
+		if( region.area > 0 && region.area < 100 )
+			list_100km.push(region);
+		if( region.area <= 0 )
+			list_zero.push(region);
+		if( region.nodes > 50000 )
+			list_100kp.push(region);
+		var size_mb = region.size * 8 / 1024 / 1024;
+		region.size_mb = size_mb;
+		if( size_mb < 1 )
+			list_1mb.push(region);
+		if( size_mb > 50 )
+			list_50mb.push(region);
+		if( size_mb > 100 )
+			list_100mb.push(region);
+		if( !/^[\x20-\x7F]*$/.test(region.name) )
+			list_bad.push(region);
+		if( region.name.indexOf(' ') >= 0 )
+			list_spaces.push(region);
+	}
+
+	statFill('names_spaces', list_spaces.length);
+	statFillList('names_bad_list', list_bad, null, 'names_bad');
+
+	list_1mb.sort(function(a, b) { return a.size_mb - b.size_mb; });
+	list_50mb.sort(function(a, b) { return a.size_mb - b.size_mb; });
+	list_100mb.sort(function(a, b) { return b.size_mb - a.size_mb; });
+	statFillList('sizes_1mb_list', list_1mb, function(r) { return formatNum(r.size_mb, 2) + ' МБ'; }, 'sizes_1mb');
+	statFillList('sizes_50mb_list', list_50mb, function(r) { return formatNum(r.size_mb, 0) + ' МБ'; }, 'sizes_50mb');
+	statFillList('sizes_100mb_list', list_100mb, function(r) { return formatNum(r.size_mb, 0) + ' МБ'; }, 'sizes_100mb');
+
+	list_100km.sort(function(a, b) { return a.area - b.area; });
+	list_100kp.sort(function(a, b) { return b.nodes - a.nodes; });
+	statFillList('areas_100km_list', list_100km, function(r) { return formatNum(r.area, 2) + ' км²'; }, 'areas_100km');
+	statFillList('areas_50k_points_list', list_100kp, 'nodes', 'areas_50k_points');
+	statFillList('areas_0_list', list_zero, null, 'areas_0');
+
+	statQuery('topo', statTopo);
+}
+
+function statTopo(data) {
+	var list_holed = [], list_multi = [], list_100km = [];
+	for( var i = 0; i < data.regions.length; i++ ) {
+		region = data.regions[i];
+		if( region.outer > 1 )
+			list_multi.push(region);
+		if( region.inner > 0 )
+			list_holed.push(region);
+		if( region.min_area > 0 && region.min_area < 100 )
+			list_100km.push(region);
+	}
+
+	list_multi.sort(function(a, b) { return b.outer - a.outer; });
+	list_holed.sort(function(a, b) { return b.inner - a.inner; });
+	list_100km.sort(function(a, b) { return a.min_area - b.min_area; });
+	statFillList('topo_holes_list', list_holed, 'inner', 'topo_holes');
+	statFillList('topo_multi_list', list_multi, 'outer', 'topo_multi');
+	statFillList('topo_100km_list', list_100km, function(r) { return formatNum(r.min_area, 2) + ' км²'; }, 'topo_100km');
+}