0.10.0: many changes

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## master branch
 
+## 0.10.0
+
+_Released 2018-06-18_
+
+* Extracted osm id encoding methods to `OsmIdCode` class.
+* Added id decoding to mwmtool.
+* Fixed printing utf-8 characters under Python 2 in mwmtool.
+* Python 2.6 is not supported officially.
+* Package `types.txt` to eliminate the need to check out omim repository.
+* `dump -s` will skip reading all the features.
+
 ## 0.9.0
 
 _Released 2017-06-08_

mwm/__init__.py

@@ -1,2 +1,3 @@
-from .mwm import MWM, Osm2Ft, __version__
+from .mwmfile import MWMFile, OsmIdCode
-from .mwmfile import MWMFile
+from .mwm import MWM, __version__
+from .osm2ft import Osm2Ft

mwm/mwm.py

@@ -1,8 +1,9 @@
 # MWM Reader Module
 from .mwmfile import MWMFile
 from datetime import datetime
+import os
 
-__version__ = '0.9.0'
+__version__ = '0.10.0'
 
 # Unprocessed sections: geomN, trgN, idx, sdx (search index),
 # addr (search address), offs (feature offsets - succinct)
@@ -20,17 +21,23 @@ class MWM(MWMFile):
                 "turn_lanes", "turn_lanes_forward", "turn_lanes_backward", "email", "postcode",
                 "wikipedia", "maxspeed", "flats", "height", "min_height",
                 "denomination", "building_levels", "test_id", "ref:sponsored", "price_rate",
-                "rating", "fuel", "routes"]
+                "rating", "banner_url", "level"]
 
-    regiondata = ["languages", "driving", "timezone", "addr_fmt", "phone_fmt", "postcode_fmt", "holidays", "housenames"]
+    regiondata = ["languages", "driving", "timezone", "addr_fmt", "phone_fmt",
+                  "postcode_fmt", "holidays", "housenames"]
 
     def __init__(self, f):
         MWMFile.__init__(self, f)
         self.read_tags()
         self.read_header()
         self.type_mapping = []
+        self.read_types(os.path.join(
+            os.getcwd(), os.path.dirname(__file__), 'types.txt'))
 
     def read_types(self, filename):
+        if not os.path.exists(filename):
+            return
+        self.type_mapping = []
         with open(filename, 'r') as ft:
             for line in ft:
                 if len(line.strip()) > 0:
@@ -268,39 +275,3 @@ class MWM(MWMFile):
                 raise Exception('Feature parsing error, read too much')
             yield feature
             self.f.seek(next_feature)
-
-
-class Osm2Ft(MWMFile):
-    def __init__(self, f, ft2osm=False, tuples=True):
-        MWMFile.__init__(self, f)
-        self.read(ft2osm, tuples)
-
-    def read(self, ft2osm=False, tuples=True):
-        """Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id."""
-        count = self.read_varuint()
-        self.data = {}
-        self.ft2osm = ft2osm
-        for i in range(count):
-            osmid = self.read_osmid(tuples)
-            fid = self.read_uint(4)
-            self.read_uint(4)  # filler
-            if osmid is not None:
-                if ft2osm:
-                    self.data[fid] = osmid
-                else:
-                    self.data[osmid] = fid
-
-    def __getitem__(self, k):
-        return self.data.get(k)
-
-    def __repr__(self):
-        return '{} with {} items'.format('ft2osm' if self.ft2osm else 'osm2ft', len(self.data))
-
-    def __len__(self):
-        return len(self.data)
-
-    def __contains__(self, k):
-        return k in self.data
-
-    def __iter__(self):
-        return iter(self.data)

mwm/mwmfile.py

@@ -3,6 +3,63 @@ import struct
 import math
 
 
+class OsmIdCode:
+    NODE = 0x4000000000000000
+    WAY = 0x8000000000000000
+    RELATION = 0xC000000000000000
+    RESET = ~(NODE | WAY | RELATION)
+
+    @staticmethod
+    def is_node(code):
+        return code & OsmIdCode.NODE == OsmIdCode.NODE
+
+    @staticmethod
+    def is_way(code):
+        return code & OsmIdCode.WAY == OsmIdCode.WAY
+
+    @staticmethod
+    def is_relation(code):
+        return code & OsmIdCode.RELATION == OsmIdCode.RELATION
+
+    @staticmethod
+    def get_type(code):
+        if OsmIdCode.is_relation(code):
+            return 'r'
+        elif OsmIdCode.is_node(code):
+            return 'n'
+        elif OsmIdCode.is_way(code):
+            return 'w'
+        return None
+
+    @staticmethod
+    def get_id(code):
+        return code & OsmIdCode.RESET
+
+    @staticmethod
+    def unpack(num):
+        typ = OsmIdCode.get_type(num)
+        if typ is None:
+            return None
+        return typ, OsmIdCode.get_id(num)
+
+    @staticmethod
+    def pack(osm_type, osm_id, int64=False):
+        if osm_type is None or len(osm_type) == 0:
+            return None
+        typ = osm_type[0].lower()
+        if typ == 'r':
+            result = osm_id | OsmIdCode.RELATION
+        elif typ == 'w':
+            result = osm_id | OsmIdCode.WAY
+        elif typ == 'n':
+            result = osm_id | OsmIdCode.NODE
+        else:
+            return None
+        if int64 and result >= 2**63:
+            result = -1 - (result ^ (2**64 - 1))
+        return result
+
+
 class MWMFile(object):
     # coding/multilang_utf8_string.cpp
     languages = ["default",
@@ -86,23 +143,12 @@ class MWMFile(object):
         AREA = 1 << 6
         POINT_EX = 3 << 5
 
-    class OsmIdCode:
-        NODE = 0x4000000000000000
-        WAY = 0x8000000000000000
-        RELATION = 0xC000000000000000
-        RESET = ~(NODE | WAY | RELATION)
-
     @staticmethod
     def unpack_osmid(num):
-        if num & MWMFile.OsmIdCode.RELATION == MWMFile.OsmIdCode.RELATION:
+        typ = OsmIdCode.get_type(num)
-            typ = 'r'
+        if typ is None:
-        elif num & MWMFile.OsmIdCode.WAY == MWMFile.OsmIdCode.WAY:
-            typ = 'w'
-        elif num & MWMFile.OsmIdCode.NODE == MWMFile.OsmIdCode.NODE:
-            typ = 'n'
-        else:
             return None
-        return typ, num & MWMFile.OsmIdCode.RESET
+        return typ, OsmIdCode.get_id(num)
 
     def read_osmid(self, as_tuple=True):
         osmid = self.read_uint(8)

mwm/mwmtool.py

@@ -1,15 +1,23 @@
 #!/usr/bin/env python
 import sys
-import os.path
 import random
 import json
 import argparse
-from . import MWM, Osm2Ft
+import re
+from . import MWM, Osm2Ft, OsmIdCode
+
+
+def print_json(data):
+    s = json.dumps(data, ensure_ascii=False, sort_keys=True)
+    if sys.version_info[0] >= 3:
+        print(s)
+    else:
+        print(s.encode('utf-8'))
 
 
 def dump_mwm(args):
     mwm = MWM(args.mwm)
-    if os.path.exists(args.types):
+    if args.types:
         mwm.read_types(args.types)
 
     print('Tags:')
@@ -20,6 +28,10 @@ def dump_mwm(args):
     print('Format: {0}, version: {1}'.format(v['fmt'], v['date'].strftime('%Y-%m-%d %H:%M')))
     print('Header: {0}'.format(mwm.read_header()))
     print('Region Info: {0}'.format(mwm.read_region_info()))
+
+    if args.short:
+        return
+
     print('Metadata count: {0}'.format(len(mwm.read_metadata())))
 
     cross = mwm.read_crossmwm()
@@ -39,13 +51,13 @@ def dump_mwm(args):
     print('Feature count: {0}'.format(i))
     print('Sample features:')
     for feature in sample:
-        print(json.dumps(feature, ensure_ascii=False))
+        print_json(feature)
 
 
 def find_feature(args):
     mwm = MWM(args.mwm)
     mwm.read_header()
-    if os.path.exists(args.types):
+    if args.types:
         mwm.read_types(args.types)
     if args.iname:
         args.iname = args.iname.lower()
@@ -75,7 +87,7 @@ def find_feature(args):
                 continue
         if args.meta and ('metadata' not in feature or args.meta not in feature['metadata']):
             continue
-        print(json.dumps(feature, ensure_ascii=False, sort_keys=True))
+        print_json(feature)
 
 
 def ft2osm(args):
@@ -84,35 +96,87 @@ def ft2osm(args):
     type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
     for ftid in args.ftid:
         if ftid in ft2osm:
-            print('https://www.openstreetmap.org/{}/{}'.format(type_abbr[ft2osm[ftid][0]], ft2osm[ftid][1]))
+            print('https://www.openstreetmap.org/{}/{}'.format(
+                type_abbr[ft2osm[ftid][0]],
+                ft2osm[ftid][1]))
         else:
             print('Could not find osm id for feature {}'.format(ftid))
             code = 2
     return code
 
 
+def decode_id(args):
+    if args.id.isdigit():
+        osm_id = OsmIdCode.unpack(int(args.id))
+        if osm_id is None:
+            print('That is not a valid identifier')
+            return 2
+        else:
+            type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
+            print('https://www.openstreetmap.org/{}/{}'.format(
+                type_abbr[osm_id[0]], osm_id[1]))
+    else:
+        m = re.search(r'/(node|way|relation)/(\d+)', args.id)
+        if m:
+            print(OsmIdCode.pack(m.group(1), int(m.group(2))))
+        else:
+            print('Please specify an URL to OSM object on its website')
+            return 2
+
+
+def dat_to_gpx(args):
+    POINT_SOURCE = ['apple', 'windows', 'android', 'google', 'tizen', 'predictor']
+    out = sys.stdout if not args.gpx else open(args.gpx, 'w')
+    # TODO
+    print('Not implemented yet, sorry.')
+    return 2
+
+
 def main():
     parser = argparse.ArgumentParser(description='Toolbox for MWM files.')
-    parser.add_argument('--types', default=os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', '..', 'data', 'types.txt'), help='path to types.txt')
+    parser.add_argument('-t', '--types', help='path to types.txt')
     subparsers = parser.add_subparsers(dest='cmd')
     subparsers.required = True
+
     parser_dump = subparsers.add_parser('dump', help='Dumps some structures.')
     parser_dump.add_argument('mwm', type=argparse.FileType('rb'), help='file to browse')
+    parser_dump.add_argument('-s', '--short', action='store_true',
+                             help='Read header only, no features')
     parser_dump.set_defaults(func=dump_mwm)
+
     parser_find = subparsers.add_parser('find', help='Finds features in a file.')
     parser_find.add_argument('mwm', type=argparse.FileType('rb'), help='file to search')
-    parser_find.add_argument('-t', dest='type', help='look inside types ("-t hwtag" will find all hwtags-*)')
+    parser_find.add_argument('-t', dest='type',
-    parser_find.add_argument('-et', dest='exact_type', help='look for a type ("-et shop won\'t find shop-chemist)')
+                             help='look inside types ("-t hwtag" will find all hwtags-*)')
-    parser_find.add_argument('-n', dest='name', help='look inside names, case-sensitive ("-n Starbucks" for all starbucks)')
+    parser_find.add_argument('-et', dest='exact_type',
-    parser_find.add_argument('-in', '-ni', dest='iname', help='look inside names, case-insensitive ("-in star" will find Starbucks)')
+                             help='look for a type ("-et shop won\'t find shop-chemist)')
-    parser_find.add_argument('-m', dest='meta', help='look for a metadata key ("m flats" for features with flats)')
+    parser_find.add_argument('-n', dest='name',
-    parser_find.add_argument('-id', dest='fid', type=int, help='look for a feature id ("-id 1234 for feature #1234)')
+                             help='look inside names, case-sensitive ("-n Starbucks" '
+                             'for all starbucks)')
+    parser_find.add_argument('-in', '-ni', dest='iname',
+                             help='look inside names, case-insensitive ("-in star" will '
+                             'find Starbucks)')
+    parser_find.add_argument('-m', dest='meta',
+                             help='look for a metadata key ("m flats" for features with flats)')
+    parser_find.add_argument('-id', dest='fid', type=int,
+                             help='look for a feature id ("-id 1234 for feature #1234)')
     parser_find.set_defaults(func=find_feature)
-    parser_osm = subparsers.add_parser('osm', help='Displays an OpenStreetMap link for a feature id.')
+
+    parser_osm = subparsers.add_parser('osm',
+                                       help='Displays an OpenStreetMap link for a feature id.')
     parser_osm.add_argument('osm2ft', type=argparse.FileType('rb'), help='.mwm.osm2ft file')
     parser_osm.add_argument('ftid', type=int, nargs='+', help='feature id')
     parser_osm.set_defaults(func=ft2osm)
 
+    parser_id = subparsers.add_parser('id', help='Decode or encode OSM ID')
+    parser_id.add_argument('id', help='MWM internal OSM ID, or a link to OSM website')
+    parser_id.set_defaults(func=decode_id)
+
+    parser_dump = subparsers.add_parser('gpx', help='Convert gps_track.dat to GPX')
+    parser_dump.add_argument('dat', type=argparse.FileType('rb'), help='file to convert')
+    parser_dump.add_argument('--gpx', '-o', type=argparse.FileType('w'), help='output gpx file')
+    parser_dump.set_defaults(func=dat_to_gpx)
+
     args = parser.parse_args()
     code = args.func(args)
     if code is not None:

mwm/osm2ft.py

@@ -0,0 +1,38 @@
+# OSM2FT Reader
+from .mwmfile import MWMFile
+
+
+class Osm2Ft(MWMFile):
+    def __init__(self, f, ft2osm=False, tuples=True):
+        MWMFile.__init__(self, f)
+        self.read(ft2osm, tuples)
+
+    def read(self, ft2osm=False, tuples=True):
+        """Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id."""
+        count = self.read_varuint()
+        self.data = {}
+        self.ft2osm = ft2osm
+        for i in range(count):
+            osmid = self.read_osmid(tuples)
+            fid = self.read_uint(4)
+            self.read_uint(4)  # filler
+            if osmid is not None:
+                if ft2osm:
+                    self.data[fid] = osmid
+                else:
+                    self.data[osmid] = fid
+
+    def __getitem__(self, k):
+        return self.data.get(k)
+
+    def __repr__(self):
+        return '{} with {} items'.format('ft2osm' if self.ft2osm else 'osm2ft', len(self.data))
+
+    def __len__(self):
+        return len(self.data)
+
+    def __contains__(self, k):
+        return k in self.data
+
+    def __iter__(self):
+        return iter(self.data)

setup.py

@@ -10,6 +10,7 @@ setup(
     author='Ilya Zverev',
     author_email='ilya@zverev.info',
     packages=['mwm'],
+    package_data={'mwm': ['types.txt']},
     url='https://github.com/mapsme/mwm.py',
     license='Apache License 2.0',
     description='Library to read binary MAPS.ME files.',
@@ -22,7 +23,6 @@ setup(
         'Environment :: Console',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
     ],