organicmaps/mwm.py
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

0.10.0: many changes

Browse source
This commit is contained in:
Ilya Zverev 2018-06-18 19:17:11 +03:00
parent 45ecc061e8
commit 8939a54cad
8 changed files with 1427 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
CHANGELOG.md
View file

@ -2,6 +2,17 @@
## master branch
## 0.10.0
_Released 2018-06-18_
* Extracted osm id encoding methods to `OsmIdCode` class.
* Added id decoding to mwmtool.
* Fixed printing utf-8 characters under Python 2 in mwmtool.
* Python 2.6 is not supported officially.
* Package `types.txt` to eliminate the need to check out omim repository.
* `dump -s` will skip reading all the features.
## 0.9.0
_Released 2017-06-08_

5
mwm/__init__.py
View file

@ -1,2 +1,3 @@
from .mwm import MWM, Osm2Ft, __version__
from .mwmfile import MWMFile
from .mwmfile import MWMFile, OsmIdCode
from .mwm import MWM, __version__
from .osm2ft import Osm2Ft

49
mwm/mwm.py
View file

@ -1,8 +1,9 @@
# MWM Reader Module
from .mwmfile import MWMFile
from datetime import datetime
import os
__version__ = '0.9.0'
__version__ = '0.10.0'
# Unprocessed sections: geomN, trgN, idx, sdx (search index),
# addr (search address), offs (feature offsets - succinct)
@ -20,17 +21,23 @@ class MWM(MWMFile):
"turn_lanes", "turn_lanes_forward", "turn_lanes_backward", "email", "postcode",
"wikipedia", "maxspeed", "flats", "height", "min_height",
"denomination", "building_levels", "test_id", "ref:sponsored", "price_rate",
"rating", "fuel", "routes"]
"rating", "banner_url", "level"]
regiondata = ["languages", "driving", "timezone", "addr_fmt", "phone_fmt", "postcode_fmt", "holidays", "housenames"]
regiondata = ["languages", "driving", "timezone", "addr_fmt", "phone_fmt",
"postcode_fmt", "holidays", "housenames"]
def __init__(self, f):
MWMFile.__init__(self, f)
self.read_tags()
self.read_header()
self.type_mapping = []
self.read_types(os.path.join(
os.getcwd(), os.path.dirname(__file__), 'types.txt'))
def read_types(self, filename):
if not os.path.exists(filename):
return
self.type_mapping = []
with open(filename, 'r') as ft:
for line in ft:
if len(line.strip()) > 0:
@ -268,39 +275,3 @@ class MWM(MWMFile):
raise Exception('Feature parsing error, read too much')
yield feature
self.f.seek(next_feature)
class Osm2Ft(MWMFile):
def __init__(self, f, ft2osm=False, tuples=True):
MWMFile.__init__(self, f)
self.read(ft2osm, tuples)
def read(self, ft2osm=False, tuples=True):
"""Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id."""
count = self.read_varuint()
self.data = {}
self.ft2osm = ft2osm
for i in range(count):
osmid = self.read_osmid(tuples)
fid = self.read_uint(4)
self.read_uint(4) # filler
if osmid is not None:
if ft2osm:
self.data[fid] = osmid
else:
self.data[osmid] = fid
def __getitem__(self, k):
return self.data.get(k)
def __repr__(self):
return '{} with {} items'.format('ft2osm' if self.ft2osm else 'osm2ft', len(self.data))
def __len__(self):
return len(self.data)
def __contains__(self, k):
return k in self.data
def __iter__(self):
return iter(self.data)

74
mwm/mwmfile.py
View file

@ -3,6 +3,63 @@ import struct
import math
class OsmIdCode:
NODE = 0x4000000000000000
WAY = 0x8000000000000000
RELATION = 0xC000000000000000
RESET = ~(NODE | WAY | RELATION)
@staticmethod
def is_node(code):
return code & OsmIdCode.NODE == OsmIdCode.NODE
@staticmethod
def is_way(code):
return code & OsmIdCode.WAY == OsmIdCode.WAY
@staticmethod
def is_relation(code):
return code & OsmIdCode.RELATION == OsmIdCode.RELATION
@staticmethod
def get_type(code):
if OsmIdCode.is_relation(code):
return 'r'
elif OsmIdCode.is_node(code):
return 'n'
elif OsmIdCode.is_way(code):
return 'w'
return None
@staticmethod
def get_id(code):
return code & OsmIdCode.RESET
@staticmethod
def unpack(num):
typ = OsmIdCode.get_type(num)
if typ is None:
return None
return typ, OsmIdCode.get_id(num)
@staticmethod
def pack(osm_type, osm_id, int64=False):
if osm_type is None or len(osm_type) == 0:
return None
typ = osm_type[0].lower()
if typ == 'r':
result = osm_id | OsmIdCode.RELATION
elif typ == 'w':
result = osm_id | OsmIdCode.WAY
elif typ == 'n':
result = osm_id | OsmIdCode.NODE
else:
return None
if int64 and result >= 2**63:
result = -1 - (result ^ (2**64 - 1))
return result
class MWMFile(object):
# coding/multilang_utf8_string.cpp
languages = ["default",
@ -86,23 +143,12 @@ class MWMFile(object):
AREA = 1 << 6
POINT_EX = 3 << 5
class OsmIdCode:
NODE = 0x4000000000000000
WAY = 0x8000000000000000
RELATION = 0xC000000000000000
RESET = ~(NODE | WAY | RELATION)
@staticmethod
def unpack_osmid(num):
if num & MWMFile.OsmIdCode.RELATION == MWMFile.OsmIdCode.RELATION:
typ = 'r'
elif num & MWMFile.OsmIdCode.WAY == MWMFile.OsmIdCode.WAY:
typ = 'w'
elif num & MWMFile.OsmIdCode.NODE == MWMFile.OsmIdCode.NODE:
typ = 'n'
else:
typ = OsmIdCode.get_type(num)
if typ is None:
return None
return typ, num & MWMFile.OsmIdCode.RESET
return typ, OsmIdCode.get_id(num)
def read_osmid(self, as_tuple=True):
osmid = self.read_uint(8)

94
mwm/mwmtool.py Executable file → Normal file
View file

@ -1,15 +1,23 @@
#!/usr/bin/env python
import sys
import os.path
import random
import json
import argparse
from . import MWM, Osm2Ft
import re
from . import MWM, Osm2Ft, OsmIdCode
def print_json(data):
s = json.dumps(data, ensure_ascii=False, sort_keys=True)
if sys.version_info[0] >= 3:
print(s)
else:
print(s.encode('utf-8'))
def dump_mwm(args):
mwm = MWM(args.mwm)
if os.path.exists(args.types):
if args.types:
mwm.read_types(args.types)
print('Tags:')
@ -20,6 +28,10 @@ def dump_mwm(args):
print('Format: {0}, version: {1}'.format(v['fmt'], v['date'].strftime('%Y-%m-%d %H:%M')))
print('Header: {0}'.format(mwm.read_header()))
print('Region Info: {0}'.format(mwm.read_region_info()))
if args.short:
return
print('Metadata count: {0}'.format(len(mwm.read_metadata())))
cross = mwm.read_crossmwm()
@ -39,13 +51,13 @@ def dump_mwm(args):
print('Feature count: {0}'.format(i))
print('Sample features:')
for feature in sample:
print(json.dumps(feature, ensure_ascii=False))
print_json(feature)
def find_feature(args):
mwm = MWM(args.mwm)
mwm.read_header()
if os.path.exists(args.types):
if args.types:
mwm.read_types(args.types)
if args.iname:
args.iname = args.iname.lower()
@ -75,7 +87,7 @@ def find_feature(args):
continue
if args.meta and ('metadata' not in feature or args.meta not in feature['metadata']):
continue
print(json.dumps(feature, ensure_ascii=False, sort_keys=True))
print_json(feature)
def ft2osm(args):
@ -84,35 +96,87 @@ def ft2osm(args):
type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
for ftid in args.ftid:
if ftid in ft2osm:
print('https://www.openstreetmap.org/{}/{}'.format(type_abbr[ft2osm[ftid][0]], ft2osm[ftid][1]))
print('https://www.openstreetmap.org/{}/{}'.format(
type_abbr[ft2osm[ftid][0]],
ft2osm[ftid][1]))
else:
print('Could not find osm id for feature {}'.format(ftid))
code = 2
return code
def decode_id(args):
if args.id.isdigit():
osm_id = OsmIdCode.unpack(int(args.id))
if osm_id is None:
print('That is not a valid identifier')
return 2
else:
type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'}
print('https://www.openstreetmap.org/{}/{}'.format(
type_abbr[osm_id[0]], osm_id[1]))
else:
m = re.search(r'/(node|way|relation)/(\d+)', args.id)
if m:
print(OsmIdCode.pack(m.group(1), int(m.group(2))))
else:
print('Please specify an URL to OSM object on its website')
return 2
def dat_to_gpx(args):
POINT_SOURCE = ['apple', 'windows', 'android', 'google', 'tizen', 'predictor']
out = sys.stdout if not args.gpx else open(args.gpx, 'w')
# TODO
print('Not implemented yet, sorry.')
return 2
def main():
parser = argparse.ArgumentParser(description='Toolbox for MWM files.')
parser.add_argument('--types', default=os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', '..', 'data', 'types.txt'), help='path to types.txt')
parser.add_argument('-t', '--types', help='path to types.txt')
subparsers = parser.add_subparsers(dest='cmd')
subparsers.required = True
parser_dump = subparsers.add_parser('dump', help='Dumps some structures.')
parser_dump.add_argument('mwm', type=argparse.FileType('rb'), help='file to browse')
parser_dump.add_argument('-s', '--short', action='store_true',
help='Read header only, no features')
parser_dump.set_defaults(func=dump_mwm)
parser_find = subparsers.add_parser('find', help='Finds features in a file.')
parser_find.add_argument('mwm', type=argparse.FileType('rb'), help='file to search')
parser_find.add_argument('-t', dest='type', help='look inside types ("-t hwtag" will find all hwtags-*)')
parser_find.add_argument('-et', dest='exact_type', help='look for a type ("-et shop won\'t find shop-chemist)')
parser_find.add_argument('-n', dest='name', help='look inside names, case-sensitive ("-n Starbucks" for all starbucks)')
parser_find.add_argument('-in', '-ni', dest='iname', help='look inside names, case-insensitive ("-in star" will find Starbucks)')
parser_find.add_argument('-m', dest='meta', help='look for a metadata key ("m flats" for features with flats)')
parser_find.add_argument('-id', dest='fid', type=int, help='look for a feature id ("-id 1234 for feature #1234)')
parser_find.add_argument('-t', dest='type',
help='look inside types ("-t hwtag" will find all hwtags-*)')
parser_find.add_argument('-et', dest='exact_type',
help='look for a type ("-et shop won\'t find shop-chemist)')
parser_find.add_argument('-n', dest='name',
help='look inside names, case-sensitive ("-n Starbucks" '
'for all starbucks)')
parser_find.add_argument('-in', '-ni', dest='iname',
help='look inside names, case-insensitive ("-in star" will '
'find Starbucks)')
parser_find.add_argument('-m', dest='meta',
help='look for a metadata key ("m flats" for features with flats)')
parser_find.add_argument('-id', dest='fid', type=int,
help='look for a feature id ("-id 1234 for feature #1234)')
parser_find.set_defaults(func=find_feature)
parser_osm = subparsers.add_parser('osm', help='Displays an OpenStreetMap link for a feature id.')
parser_osm = subparsers.add_parser('osm',
help='Displays an OpenStreetMap link for a feature id.')
parser_osm.add_argument('osm2ft', type=argparse.FileType('rb'), help='.mwm.osm2ft file')
parser_osm.add_argument('ftid', type=int, nargs='+', help='feature id')
parser_osm.set_defaults(func=ft2osm)
parser_id = subparsers.add_parser('id', help='Decode or encode OSM ID')
parser_id.add_argument('id', help='MWM internal OSM ID, or a link to OSM website')
parser_id.set_defaults(func=decode_id)
parser_dump = subparsers.add_parser('gpx', help='Convert gps_track.dat to GPX')
parser_dump.add_argument('dat', type=argparse.FileType('rb'), help='file to convert')
parser_dump.add_argument('--gpx', '-o', type=argparse.FileType('w'), help='output gpx file')
parser_dump.set_defaults(func=dat_to_gpx)
args = parser.parse_args()
code = args.func(args)
if code is not None:

38
mwm/osm2ft.py Normal file
View file

@ -0,0 +1,38 @@
# OSM2FT Reader
from .mwmfile import MWMFile
class Osm2Ft(MWMFile):
def __init__(self, f, ft2osm=False, tuples=True):
MWMFile.__init__(self, f)
self.read(ft2osm, tuples)
def read(self, ft2osm=False, tuples=True):
"""Reads mwm.osm2ft file, returning a dict of feature id <-> osm way id."""
count = self.read_varuint()
self.data = {}
self.ft2osm = ft2osm
for i in range(count):
osmid = self.read_osmid(tuples)
fid = self.read_uint(4)
self.read_uint(4) # filler
if osmid is not None:
if ft2osm:
self.data[fid] = osmid
else:
self.data[osmid] = fid
def __getitem__(self, k):
return self.data.get(k)
def __repr__(self):
return '{} with {} items'.format('ft2osm' if self.ft2osm else 'osm2ft', len(self.data))
def __len__(self):
return len(self.data)
def __contains__(self, k):
return k in self.data
def __iter__(self):
return iter(self.data)

1225
mwm/types.txt Normal file
View file

File diff suppressed because it is too large Load diff

2
setup.py
View file

@ -10,6 +10,7 @@ setup(
author='Ilya Zverev',
author_email='ilya@zverev.info',
packages=['mwm'],
package_data={'mwm': ['types.txt']},
url='https://github.com/mapsme/mwm.py',
license='Apache License 2.0',
description='Library to read binary MAPS.ME files.',
@ -22,7 +23,6 @@ setup(
'Environment :: Console',
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python',
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
],