diff --git a/tools/python/local_ads/features_db_updater.py b/tools/python/local_ads/features_db_updater.py new file mode 100755 index 0000000000..472a37a43a --- /dev/null +++ b/tools/python/local_ads/features_db_updater.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function + +import os +import sys + +# TODO(mgsergio, zveric, yershov): Make mwm an installable module. +sys.path.append( + os.path.join( + os.path.dirname(__file__), '..', 'mwm' + ) +) + +import argparse +import csv +# c_long is used to get signed int64. Postgres can't handle uint64. +import ctypes +import logging +import mwm + +from itertools import islice +from zlib import adler32 + + +def get_args(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '--mapping_names', + nargs='+', + help='osm2ft files to handle.' + ) + group.add_argument( + '--mapping_path', + nargs=1, + action=AppendOsm2FidAction, + dest='mapping_names', + help='Path to folder with .osm2ft. Each file whould be handled.' + ) + + parser.add_argument( + '--version', + required=True, + type=int, + help='The version of mwm for which a mapping is generated.' + ) + parser.add_argument( + '--head', + type=int, + help='Write that much lines of osmid <-> fid to stdout.' + ) + + return parser.parse_args(); + + +def get_mapping(mapping_name): + with open(mapping_name, 'rb') as f: + osm2ft = mwm.read_osm2ft(f, tuples=False) + + for osmid, fid in osm2ft.iteritems(): + yield ctypes.c_long(osmid).value, fid + + +def print_mapping(mapping, count): + for osmid, fid in islice(mapping, count): + print('{}\t{}'.format(osmid, fid)) + +def generate_id_from_name_and_version(name, version): + return ctypes.c_long((adler32(name) << 32) | version).value + + +def generate_csvs(mapping, mapping_name, version): + mwm_id = generate_id_from_name_and_version( + mapping_name, + version + ) + + with open('mwm.csv', 'wb') as f: + w = csv.writer(f) + w.writerow(['id', 'name', 'version']) + w.writerow([ + mwm_id, + mapping_name, + version, + ]) + with open('mapping.csv', 'wb') as f: + w = csv.writer(f) + w.writerow(['osmid', 'fid', 'mwm_id']) + for row in mapping: + w.writerow(row + (mwm_id, )) + + +def main(): + args = get_args() + for mapping_name in args.mapping_names: + mapping = get_mapping(mapping_name) + if args.head: + print('{}:'.format(mapping_name)) + print_mapping(mapping, args.head) + exit(0) + mwm_name = ( + os.path.basename(mapping_name) + .split('.', 1) + )[0] + generate_csvs( + mapping, + mwm_name, + args.version + ) + + +class AppendOsm2FidAction(argparse.Action): + def __init__(self, option_strings, dest, nargs=None, **kwargs): + assert nargs == 1, 'nargs should equals to 1.' + super(AppendOsm2FidAction, self).__init__( + option_strings, + dest, + nargs=1, + **kwargs + ) + + def __call__(self, parser, namespace, values, option_string=None): + values = [ + os.path.join(values[0], mapping_name) + for mapping_name in os.listdir(values[0]) + if mapping_name.endswith('.osm2ft') + ] + setattr(namespace, self.dest, values) + + +if __name__ == '__main__': + main() diff --git a/tools/python/mwm/ft2osm.py b/tools/python/mwm/ft2osm.py index e8620b5fbf..c2dd1985e1 100755 --- a/tools/python/mwm/ft2osm.py +++ b/tools/python/mwm/ft2osm.py @@ -8,14 +8,17 @@ if len(sys.argv) < 3: sys.exit(1) with open(sys.argv[1], 'rb') as f: - ft2osm = mwm.read_osm2ft(f, True) + ft2osm = mwm.read_osm2ft(f, ft2osm=True) code = 0 +type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'} for ftid in sys.argv[2:]: ftid = int(ftid) if ftid in ft2osm: - type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'} - print('https://www.openstreetmap.org/{}/{}'.format(type_abbr[ft2osm[ftid][0]], ft2osm[ftid][1])) + print('https://www.openstreetmap.org/{}/{}'.format( + type_abbr[ft2osm[ftid][0]], + ft2osm[ftid][1] + )) else: print('Could not find osm id for feature {}'.format(ftid)) code = 2 diff --git a/tools/python/mwm/mwm.py b/tools/python/mwm/mwm.py index 5848e1afd1..e67ae8def0 100644 --- a/tools/python/mwm/mwm.py +++ b/tools/python/mwm/mwm.py @@ -92,7 +92,7 @@ class MWM: result = {} coord_bits = self.read_varuint() self.coord_size = (1 << coord_bits) - 1 - self.base_point = self.mwm_bitwise_split(self.read_varuint()) + self.base_point = mwm_bitwise_split(self.read_varuint()) result['basePoint'] = self.to_4326(self.base_point) result['bounds'] = self.read_bounds() result['scales'] = self.read_uint_array() @@ -333,31 +333,13 @@ class MWM: def read_varint(self): return read_varint(self.f) - def mwm_unshuffle(self, x): - x = ((x & 0x22222222) << 1) | ((x >> 1) & 0x22222222) | (x & 0x99999999) - x = ((x & 0x0C0C0C0C) << 2) | ((x >> 2) & 0x0C0C0C0C) | (x & 0xC3C3C3C3) - x = ((x & 0x00F000F0) << 4) | ((x >> 4) & 0x00F000F0) | (x & 0xF00FF00F) - x = ((x & 0x0000FF00) << 8) | ((x >> 8) & 0x0000FF00) | (x & 0xFF0000FF) - return x - - def mwm_bitwise_split(self, v): - hi = self.mwm_unshuffle(v >> 32) - lo = self.mwm_unshuffle(v & 0xFFFFFFFF) - x = ((hi & 0xFFFF) << 16) | (lo & 0xFFFF) - y = (hi & 0xFFFF0000) | (lo >> 16) - return (x, y) - - def mwm_decode_delta(self, v, ref): - x, y = self.mwm_bitwise_split(v) - return ref[0] + zigzag_decode(x), ref[1] + zigzag_decode(y) - def read_point(self, ref, packed=True): """Reads an unsigned point, returns (x, y).""" if packed: u = self.read_varuint() else: u = self.read_uint(8) - return self.mwm_decode_delta(u, ref) + return mwm_decode_delta(u, ref) def to_4326(self, point): """Convert a point in maps.me-mercator CS to WGS-84 (EPSG:4326).""" @@ -376,8 +358,8 @@ class MWM: def read_bounds(self): """Reads mercator bounds, returns (min_lon, min_lat, max_lon, max_lat).""" - rmin = self.mwm_bitwise_split(self.read_varint()) - rmax = self.mwm_bitwise_split(self.read_varint()) + rmin = mwm_bitwise_split(self.read_varint()) + rmax = mwm_bitwise_split(self.read_varint()) pmin = self.to_4326(rmin) pmax = self.to_4326(rmax) return (pmin[0], pmin[1], pmax[0], pmax[1]) @@ -443,6 +425,27 @@ class MWM: return langs +def mwm_unshuffle(x): + x = ((x & 0x22222222) << 1) | ((x >> 1) & 0x22222222) | (x & 0x99999999) + x = ((x & 0x0C0C0C0C) << 2) | ((x >> 2) & 0x0C0C0C0C) | (x & 0xC3C3C3C3) + x = ((x & 0x00F000F0) << 4) | ((x >> 4) & 0x00F000F0) | (x & 0xF00FF00F) + x = ((x & 0x0000FF00) << 8) | ((x >> 8) & 0x0000FF00) | (x & 0xFF0000FF) + return x + + +def mwm_bitwise_split(v): + hi = mwm_unshuffle(v >> 32) + lo = mwm_unshuffle(v & 0xFFFFFFFF) + x = ((hi & 0xFFFF) << 16) | (lo & 0xFFFF) + y = (hi & 0xFFFF0000) | (lo >> 16) + return (x, y) + + +def mwm_decode_delta(v, ref): + x, y = mwm_bitwise_split(v) + return ref[0] + zigzag_decode(x), ref[1] + zigzag_decode(y) + + def read_uint(f, bytelen=1): if bytelen == 1: fmt = 'B' @@ -500,15 +503,19 @@ def unpack_osmid(num): typ = 'r' else: return None - return (typ, num & RESET) + return typ, num & RESET -def read_osm2ft(f, ft2osm=False): - """Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id.""" +# TODO(zverik, mgsergio): Move this to a separate module, cause it has nothing +# to do with mwm. +def read_osm2ft(f, ft2osm=False, tuples=True): + """Reads mwm.osm2ft file, returning a dict of feature id <-> osm id.""" count = read_varuint(f) result = {} for i in range(count): - osmid = unpack_osmid(read_uint(f, 8)) + osmid = read_uint(f, 8) + if tuples: + osmid = unpack_osmid(osmid) fid = read_uint(f, 4) read_uint(f, 4) # filler if osmid is not None: