Merge pull request #5788 from mgsergio/localads-db-updater

Add a script to generate mappings in csv.

tools/python/local_ads/features_db_updater.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python2.7
+
+from __future__ import print_function
+
+import os
+import sys
+
+# TODO(mgsergio, zveric, yershov): Make mwm an installable module.
+sys.path.append(
+    os.path.join(
+        os.path.dirname(__file__), '..', 'mwm'
+    )
+)
+
+import argparse
+import csv
+# c_long is used to get signed int64. Postgres can't handle uint64.
+import ctypes
+import logging
+import mwm
+
+from itertools import islice
+from zlib import adler32
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        '--mapping_names',
+        nargs='+',
+        help='osm2ft files to handle.'
+    )
+    group.add_argument(
+        '--mapping_path',
+        nargs=1,
+        action=AppendOsm2FidAction,
+        dest='mapping_names',
+        help='Path to folder with .osm2ft. Each file whould be handled.'
+    )
+
+    parser.add_argument(
+        '--version',
+        required=True,
+        type=int,
+        help='The version of mwm for which a mapping is generated.'
+    )
+    parser.add_argument(
+        '--head',
+        type=int,
+        help='Write that much lines of osmid <-> fid to stdout.'
+    )
+
+    return parser.parse_args();
+
+
+def get_mapping(mapping_name):
+    with open(mapping_name, 'rb') as f:
+        osm2ft = mwm.read_osm2ft(f, tuples=False)
+
+    for osmid, fid in osm2ft.iteritems():
+        yield ctypes.c_long(osmid).value, fid
+
+
+def print_mapping(mapping, count):
+    for osmid, fid in islice(mapping, count):
+        print('{}\t{}'.format(osmid, fid))
+
+def generate_id_from_name_and_version(name, version):
+    return ctypes.c_long((adler32(name) << 32) | version).value
+
+
+def generate_csvs(mapping, mapping_name, version):
+    mwm_id = generate_id_from_name_and_version(
+        mapping_name,
+        version
+    )
+
+    with open('mwm.csv', 'wb') as f:
+        w = csv.writer(f)
+        w.writerow(['id', 'name', 'version'])
+        w.writerow([
+            mwm_id,
+            mapping_name,
+            version,
+        ])
+    with open('mapping.csv', 'wb') as f:
+        w = csv.writer(f)
+        w.writerow(['osmid', 'fid', 'mwm_id'])
+        for row in mapping:
+            w.writerow(row + (mwm_id, ))
+
+
+def main():
+    args = get_args()
+    for mapping_name in args.mapping_names:
+        mapping = get_mapping(mapping_name)
+        if args.head:
+            print('{}:'.format(mapping_name))
+            print_mapping(mapping, args.head)
+            exit(0)
+        mwm_name = (
+            os.path.basename(mapping_name)
+            .split('.', 1)
+        )[0]
+        generate_csvs(
+            mapping,
+            mwm_name,
+            args.version
+        )
+
+
+class AppendOsm2FidAction(argparse.Action):
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        assert nargs == 1, 'nargs should equals to 1.'
+        super(AppendOsm2FidAction, self).__init__(
+            option_strings,
+            dest,
+            nargs=1,
+            **kwargs
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        values = [
+            os.path.join(values[0], mapping_name)
+            for mapping_name in os.listdir(values[0])
+            if mapping_name.endswith('.osm2ft')
+        ]
+        setattr(namespace, self.dest, values)
+
+
+if __name__ == '__main__':
+    main()

tools/python/mwm/ft2osm.py

@@ -8,14 +8,17 @@ if len(sys.argv) < 3:
     sys.exit(1)
 
 with open(sys.argv[1], 'rb') as f:
-    ft2osm = mwm.read_osm2ft(f, True)
+    ft2osm = mwm.read_osm2ft(f, ft2osm=True)
 
 code = 0
+type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
 for ftid in sys.argv[2:]:
     ftid = int(ftid)
     if ftid in ft2osm:
-        type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
-        print('https://www.openstreetmap.org/{}/{}'.format(type_abbr[ft2osm[ftid][0]], ft2osm[ftid][1]))
+        print('https://www.openstreetmap.org/{}/{}'.format(
+            type_abbr[ft2osm[ftid][0]],
+            ft2osm[ftid][1]
+        ))
     else:
         print('Could not find osm id for feature {}'.format(ftid))
         code = 2

tools/python/mwm/mwm.py

@@ -92,7 +92,7 @@ class MWM:
         result = {}
         coord_bits = self.read_varuint()
         self.coord_size = (1 << coord_bits) - 1
-        self.base_point = self.mwm_bitwise_split(self.read_varuint())
+        self.base_point = mwm_bitwise_split(self.read_varuint())
         result['basePoint'] = self.to_4326(self.base_point)
         result['bounds'] = self.read_bounds()
         result['scales'] = self.read_uint_array()
@@ -333,31 +333,13 @@ class MWM:
     def read_varint(self):
         return read_varint(self.f)
 
-    def mwm_unshuffle(self, x):
-        x = ((x & 0x22222222) << 1) | ((x >> 1) & 0x22222222) | (x & 0x99999999)
-        x = ((x & 0x0C0C0C0C) << 2) | ((x >> 2) & 0x0C0C0C0C) | (x & 0xC3C3C3C3)
-        x = ((x & 0x00F000F0) << 4) | ((x >> 4) & 0x00F000F0) | (x & 0xF00FF00F)
-        x = ((x & 0x0000FF00) << 8) | ((x >> 8) & 0x0000FF00) | (x & 0xFF0000FF)
-        return x
-
-    def mwm_bitwise_split(self, v):
-        hi = self.mwm_unshuffle(v >> 32)
-        lo = self.mwm_unshuffle(v & 0xFFFFFFFF)
-        x = ((hi & 0xFFFF) << 16) | (lo & 0xFFFF)
-        y =     (hi & 0xFFFF0000) | (lo >> 16)
-        return (x, y)
-
-    def mwm_decode_delta(self, v, ref):
-        x, y = self.mwm_bitwise_split(v)
-        return ref[0] + zigzag_decode(x), ref[1] + zigzag_decode(y)
-
     def read_point(self, ref, packed=True):
         """Reads an unsigned point, returns (x, y)."""
         if packed:
             u = self.read_varuint()
         else:
             u = self.read_uint(8)
-        return self.mwm_decode_delta(u, ref)
+        return mwm_decode_delta(u, ref)
 
     def to_4326(self, point):
         """Convert a point in maps.me-mercator CS to WGS-84 (EPSG:4326)."""
@@ -376,8 +358,8 @@ class MWM:
 
     def read_bounds(self):
         """Reads mercator bounds, returns (min_lon, min_lat, max_lon, max_lat)."""
-        rmin = self.mwm_bitwise_split(self.read_varint())
-        rmax = self.mwm_bitwise_split(self.read_varint())
+        rmin = mwm_bitwise_split(self.read_varint())
+        rmax = mwm_bitwise_split(self.read_varint())
         pmin = self.to_4326(rmin)
         pmax = self.to_4326(rmax)
         return (pmin[0], pmin[1], pmax[0], pmax[1])
@@ -443,6 +425,27 @@ class MWM:
         return langs
 
 
+def mwm_unshuffle(x):
+    x = ((x & 0x22222222) << 1) | ((x >> 1) & 0x22222222) | (x & 0x99999999)
+    x = ((x & 0x0C0C0C0C) << 2) | ((x >> 2) & 0x0C0C0C0C) | (x & 0xC3C3C3C3)
+    x = ((x & 0x00F000F0) << 4) | ((x >> 4) & 0x00F000F0) | (x & 0xF00FF00F)
+    x = ((x & 0x0000FF00) << 8) | ((x >> 8) & 0x0000FF00) | (x & 0xFF0000FF)
+    return x
+
+
+def mwm_bitwise_split(v):
+    hi = mwm_unshuffle(v >> 32)
+    lo = mwm_unshuffle(v & 0xFFFFFFFF)
+    x = ((hi & 0xFFFF) << 16) | (lo & 0xFFFF)
+    y =     (hi & 0xFFFF0000) | (lo >> 16)
+    return (x, y)
+
+
+def mwm_decode_delta(v, ref):
+    x, y = mwm_bitwise_split(v)
+    return ref[0] + zigzag_decode(x), ref[1] + zigzag_decode(y)
+
+
 def read_uint(f, bytelen=1):
     if bytelen == 1:
         fmt = 'B'
@@ -500,15 +503,19 @@ def unpack_osmid(num):
         typ = 'r'
     else:
         return None
-    return (typ, num & RESET)
+    return typ, num & RESET
 
 
-def read_osm2ft(f, ft2osm=False):
-    """Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id."""
+# TODO(zverik, mgsergio): Move this to a separate module, cause it has nothing
+# to do with mwm.
+def read_osm2ft(f, ft2osm=False, tuples=True):
+    """Reads mwm.osm2ft file, returning a dict of feature id <-> osm id."""
     count = read_varuint(f)
     result = {}
     for i in range(count):
-        osmid = unpack_osmid(read_uint(f, 8))
+        osmid = read_uint(f, 8)
+        if tuples:
+            osmid = unpack_osmid(osmid)
         fid = read_uint(f, 4)
         read_uint(f, 4)  # filler
         if osmid is not None: