From 550eceb33f3436b1e20d0b3789f4bd341fedd97e Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Mon, 6 May 2019 12:25:40 +0300 Subject: [PATCH] [scripts] Fixed import errors. --- tools/python/booking/__main__.py | 51 ++++++++ tools/python/booking/download_hotels.py | 49 +------- tools/python/descriptions/__main__.py | 52 +++++++++ .../descriptions/descriptions_downloader.py | 45 -------- tools/python/mwm/__main__.py | 109 ++++++++++++++++++ tools/python/mwm/decode_id.py | 41 +++---- tools/python/mwm/dump_mwm.py | 61 +++++----- tools/python/mwm/find_feature.py | 59 ++++------ tools/python/mwm/ft2osm.py | 26 +---- tools/python/mwm/mwm_feature_compare.py | 28 +---- tools/unix/generate_planet.sh | 11 +- 11 files changed, 301 insertions(+), 231 deletions(-) create mode 100644 tools/python/booking/__main__.py create mode 100644 tools/python/descriptions/__main__.py create mode 100644 tools/python/mwm/__main__.py diff --git a/tools/python/booking/__main__.py b/tools/python/booking/__main__.py new file mode 100644 index 0000000000..eb62827f14 --- /dev/null +++ b/tools/python/booking/__main__.py @@ -0,0 +1,51 @@ +import argparse +import datetime +import logging +import os +import sys + +from tqdm import tqdm + +from .api.booking_api import LIMIT_REQUESTS_PER_MINUTE +from .download_hotels import download + + +def process_options(): + parser = argparse.ArgumentParser(description="Download and process booking hotels.") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("--logfile", default="", + help="Name and destination for log file") + parser.add_argument("--password", required=True, dest="password", + help="Booking.com account password") + parser.add_argument("--user", required=True, dest="user", + help="Booking.com account user name") + parser.add_argument("--threads_count", default=1, type=int, + help="The number of threads for processing countries.") + parser.add_argument("--output", required=True, dest="output", + help="Name and destination for output file") + parser.add_argument("--country_code", default=None, action="append", + help="Download hotels of this country.") + options = parser.parse_args() + return options + + +def main(): + options = process_options() + logfile = "" + if options.logfile: + logfile = options.logfile + else: + now = datetime.datetime.now() + name = f"{now.strftime('%d_%m_%Y-%H_%M_%S')}_booking_hotels.log" + logfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), name) + print(f"Logs saved to {logfile}.", file=sys.stdout) + if options.threads_count > 1: + print(f"Limit requests per minute is {LIMIT_REQUESTS_PER_MINUTE}.", file=sys.stdout) + logging.basicConfig(level=logging.DEBUG, filename=logfile, + format="%(thread)d [%(asctime)s] %(levelname)s: %(message)s") + with tqdm(disable=not options.verbose) as progress_bar: + download(options.country_code, options.user, options.password, + options.output, options.threads_count, progress_bar) + + +main() diff --git a/tools/python/booking/download_hotels.py b/tools/python/booking/download_hotels.py index 1374459a61..245858bc0a 100755 --- a/tools/python/booking/download_hotels.py +++ b/tools/python/booking/download_hotels.py @@ -1,10 +1,5 @@ -#!/usr/bin/env python -import argparse -import datetime import logging -import os import statistics -import sys from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from functools import partial @@ -14,7 +9,7 @@ import math from eviltransform import gcj2wgs_exact from tqdm import tqdm -from .api.booking_api import BookingApi, BookingListApi, LIMIT_REQUESTS_PER_MINUTE +from .api.booking_api import BookingApi, BookingListApi from .api.exceptions import GettingMinPriceError SUPPORTED_LANGUAGES = ("en", "ru", "ar", "cs", "da", "nl", "fi", "fr", "de", @@ -194,45 +189,3 @@ def download(country_code, user, password, path, threads_count, f.writelines([f"{x}\n" for x in lines]) progress_bar.update() logging.info(f"Hotels were saved to {path}.") - - -def process_options(): - parser = argparse.ArgumentParser(description="Download and process booking hotels.") - parser.add_argument("-v", "--verbose", action="store_true") - parser.add_argument("--logfile", default="", - help="Name and destination for log file") - parser.add_argument("--password", required=True, dest="password", - help="Booking.com account password") - parser.add_argument("--user", required=True, dest="user", - help="Booking.com account user name") - parser.add_argument("--threads_count", default=1, type=int, - help="The number of threads for processing countries.") - parser.add_argument("--output", required=True, dest="output", - help="Name and destination for output file") - parser.add_argument("--country_code", default=None, action="append", - help="Download hotels of this country.") - options = parser.parse_args() - return options - - -def main(): - options = process_options() - logfile = "" - if options.logfile: - logfile = options.logfile - else: - now = datetime.datetime.now() - name = f"{now.strftime('%d_%m_%Y-%H_%M_%S')}_booking_hotels.log" - logfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), name) - print(f"Logs saved to {logfile}.", file=sys.stdout) - if options.threads_count > 1: - print(f"Limit requests per minute is {LIMIT_REQUESTS_PER_MINUTE}.", file=sys.stdout) - logging.basicConfig(level=logging.DEBUG, filename=logfile, - format="%(thread)d [%(asctime)s] %(levelname)s: %(message)s") - with tqdm(disable=not options.verbose) as progress_bar: - download(options.country_code, options.user, options.password, - options.output, options.threads_count, progress_bar) - - -if __name__ == "__main__": - main() diff --git a/tools/python/descriptions/__main__.py b/tools/python/descriptions/__main__.py new file mode 100644 index 0000000000..e1bd1194bb --- /dev/null +++ b/tools/python/descriptions/__main__.py @@ -0,0 +1,52 @@ +import argparse +import itertools +import logging +import os + +import wikipediaapi + +from .descriptions_downloader import (log, check_and_get_checker, + download_from_wikipedia_tags, + download_from_wikidata_tags) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Download wiki pages.") + parser.add_argument("--output_dir", metavar="PATH", type=str, + help="Output dir for saving pages") + parser.add_argument("--popularity", metavar="PATH", type=str, + help="File with popular object ids for which we " + "download wikipedia data. If not given, download " + "for all objects.") + parser.add_argument('--wikipedia', metavar="PATH", type=str, required=True, + help="Input file with wikipedia url.") + parser.add_argument('--wikidata', metavar="PATH", type=str, + help="Input file with wikidata ids.") + parser.add_argument('--langs', metavar="LANGS", type=str, nargs='+', + action='append', + help="Languages ​​for pages. If left blank, pages in all " + "available languages ​​will be loaded.") + return parser.parse_args() + + +def main(): + log.setLevel(logging.WARNING) + wikipediaapi.log.setLevel(logging.WARNING) + args = parse_args() + wikipedia_file = args.wikipedia + wikidata_file = args.wikidata + output_dir = args.output_dir + popularity_file = args.popularity + langs = list(itertools.chain.from_iterable(args.langs)) + os.makedirs(output_dir, exist_ok=True) + checker = check_and_get_checker(popularity_file) + download_from_wikipedia_tags(wikipedia_file, output_dir, langs, checker) + if wikidata_file is None: + log.warning(f"Wikidata file not set.") + elif os.path.exists(wikidata_file): + download_from_wikidata_tags(wikidata_file, output_dir, langs, checker) + else: + log.warning(f"Wikidata ({wikidata_file}) file not set.") + + +main() \ No newline at end of file diff --git a/tools/python/descriptions/descriptions_downloader.py b/tools/python/descriptions/descriptions_downloader.py index 9471608066..b9e08be9b9 100644 --- a/tools/python/descriptions/descriptions_downloader.py +++ b/tools/python/descriptions/descriptions_downloader.py @@ -1,6 +1,4 @@ -import argparse import functools -import itertools import json import logging import os @@ -279,46 +277,3 @@ def check_and_get_checker(popularity_file): else: log.error(f"Popularity file ({popularity_file}) not found.") return should_download_page(popularity_set) - - -def parse_args(): - parser = argparse.ArgumentParser(description="Download wiki pages.") - parser.add_argument("--output_dir", metavar="PATH", type=str, - help="Output dir for saving pages") - parser.add_argument("--popularity", metavar="PATH", type=str, - help="File with popular object ids for which we " - "download wikipedia data. If not given, download " - "for all objects.") - parser.add_argument('--wikipedia', metavar="PATH", type=str, required=True, - help="Input file with wikipedia url.") - parser.add_argument('--wikidata', metavar="PATH", type=str, - help="Input file with wikidata ids.") - parser.add_argument('--langs', metavar="LANGS", type=str, nargs='+', - action='append', - help="Languages ​​for pages. If left blank, pages in all " - "available languages ​​will be loaded.") - return parser.parse_args() - - -def main(): - log.setLevel(logging.WARNING) - wikipediaapi.log.setLevel(logging.WARNING) - args = parse_args() - wikipedia_file = args.wikipedia - wikidata_file = args.wikidata - output_dir = args.output_dir - popularity_file = args.popularity - langs = list(itertools.chain.from_iterable(args.langs)) - os.makedirs(output_dir, exist_ok=True) - checker = check_and_get_checker(popularity_file) - download_from_wikipedia_tags(wikipedia_file, output_dir, langs, checker) - if wikidata_file is None: - log.warning(f"Wikidata file not set.") - elif os.path.exists(wikidata_file): - download_from_wikidata_tags(wikidata_file, output_dir, langs, checker) - else: - log.warning(f"Wikidata ({wikidata_file}) file not set.") - - -if __name__ == "__main__": - main() diff --git a/tools/python/mwm/__main__.py b/tools/python/mwm/__main__.py new file mode 100644 index 0000000000..e480e775c8 --- /dev/null +++ b/tools/python/mwm/__main__.py @@ -0,0 +1,109 @@ +import argparse +import sys + +from .decode_id import decode_id +from .dump_mwm import dump_mwm +from .find_feature import find_feature +from .ft2osm import ft2osm +from .mwm_feature_compare import compare_mwm + + +class Mwm: + def __init__(self): + parser = argparse.ArgumentParser( + description="Mwm utils", + usage="""mwm [] +The most commonly used mwm commands are: + decode_id Unpacks maps.me OSM id to an OSM object link. + dump_mwm Dumps some MWM structures. + find_feature Finds features in an mwm file based on a query. + ft2osm Finds an OSM object for a given feature id. + mwm_feature_compare Compares feature count in .mwm files. + """) + parser.add_argument("command", help="Subcommand to run") + args = parser.parse_args(sys.argv[1:2]) + if not hasattr(self, args.command): + print(f"Unrecognized command {args.command}") + parser.print_help() + exit(1) + getattr(self, args.command)() + + @staticmethod + def decode_id(): + parser = argparse.ArgumentParser( + description="Unpacks maps.me OSM id to an OSM object link.") + parser.add_argument("--id", type=str, required=True, + help="OsmId or url from osm.org.") + args = parser.parse_args(sys.argv[2:]) + id = decode_id(args.id) + if id is None: + print("Decode id error.") + exit(1) + print(id) + + @staticmethod + def dump_mwm(): + parser = argparse.ArgumentParser( + description="Dumps some MWM structures.") + parser.add_argument("--path", type=str, required=True, + help="Path to mwm.") + args = parser.parse_args(sys.argv[2:]) + dump_mwm(args.path) + + @staticmethod + def find_method(): + parser = argparse.ArgumentParser( + description="Finds features in an mwm file based on a query.") + parser.add_argument("--path", type=str, required=True, + help="Path to mwm.") + parser.add_argument("--type", type=str, required=True, + choices=["t", "et", "n", "m", "id"], + help='''Type: + t for inside types ("t hwtag" will find all hwtags-*) + et for exact type ("et shop" won\'t find shop-chemist) + n for names, case-sensitive ("n Starbucks" fo r all starbucks) + m for metadata keys ("m flats" for features with flats + id for feature id ("id 1234" for feature #1234''') + parser.add_argument("--str", type=str, required=True, + help="String to find in mwm") + args = parser.parse_args(sys.argv[2:]) + find_feature(args.path, args.type, args.str) + + @staticmethod + def ft2osm(): + parser = argparse.ArgumentParser( + description="Finds features in an mwm file based on a query.") + parser.add_argument("--path", type=str, required=True, + help="Path to osm to feature mapping.") + parser.add_argument("--id", type=str, required=True, + help="Feature id.") + args = parser.parse_args(sys.argv[2:]) + id = ft2osm(args.path, args.id) + if id is None: + print("Error: id not found.") + exit(1) + print(id) + + @staticmethod + def mwm_feature_compare(): + parser = argparse.ArgumentParser( + description="Compares feature count in .mwm files.") + parser.add_argument("-n", "--new", help="New mwm files path", + type=str, required=True) + parser.add_argument("-o", "--old", help="Old mwm files path", + type=str, required=True) + parser.add_argument("-f", "--feature", help="Feature name to count", + type=str, required=True) + parser.add_argument("-t", "--threshold", + help="Threshold in percent to warn", type=int, + default=20) + + args = parser.parse_args() + if not compare_mwm(args.old, args.new, args.feature, + args.threshold): + print( + "Warning: some .mwm files lost more than {}% booking hotels".format( + args.threshold)) + + +Mwm() diff --git a/tools/python/mwm/decode_id.py b/tools/python/mwm/decode_id.py index c362143ab4..a910dbc6a0 100755 --- a/tools/python/mwm/decode_id.py +++ b/tools/python/mwm/decode_id.py @@ -1,30 +1,23 @@ -#!/usr/bin/env python import re -import sys from . import mwm -if len(sys.argv) < 2: - print('This script unpacks maps.me OSM id to an OSM object link.') - print('Usage: {} { | }'.format(sys.argv[0])) - sys.exit(1) -if sys.argv[1].isdigit(): - osm_id = mwm.unpack_osmid(int(sys.argv[1])) - type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'} - print('https://www.openstreetmap.org/{}/{}'.format( - type_abbr[osm_id[0]], osm_id[1])) -else: - m = re.search(r'/(node|way|relation)/(\d+)', sys.argv[1]) - if m: - oid = int(m.group(2)) - if m.group(1) == 'node': - oid |= mwm.OsmIdCode.NODE - elif m.group(1) == 'way': - oid |= mwm.OsmIdCode.WAY - elif m.group(1) == 'relation': - oid |= mwm.OsmIdCode.RELATION - print(oid) +def decode_id(id): + if id.isdigit(): + osm_id = mwm.unpack_osmid(int(id)) + type_abbr = {"n": "node", "w": "way", "r": "relation"} + return f"https://www.openstreetmap.org/{type_abbr[osm_id[0]]}/{osm_id[1]}" else: - print('Unknown parameter format') - sys.exit(2) + m = re.search(r"/(node|way|relation)/(\d+)", id) + if m: + oid = int(m.group(2)) + if m.group(1) == "node": + oid |= mwm.OsmIdCode.NODE + elif m.group(1) == "way": + oid |= mwm.OsmIdCode.WAY + elif m.group(1) == "relation": + oid |= mwm.OsmIdCode.RELATION + return oid + else: + return None diff --git a/tools/python/mwm/dump_mwm.py b/tools/python/mwm/dump_mwm.py index c8ba052085..db64c24771 100755 --- a/tools/python/mwm/dump_mwm.py +++ b/tools/python/mwm/dump_mwm.py @@ -1,4 +1,3 @@ -#!/usr/bin/python import json import os.path import random @@ -6,38 +5,36 @@ import sys from .mwm import MWM -if len(sys.argv) < 2: - print('Dumps some MWM structures.') - print('Usage: {0} '.format(sys.argv[0])) - sys.exit(1) -mwm = MWM(open(sys.argv[1], 'rb')) -mwm.read_types(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'data', 'types.txt')) -print('Tags:') -tvv = sorted([(k, v[0], v[1]) for k, v in mwm.tags.items()], key=lambda x: x[1]) -for tv in tvv: - print(' {0:<8}: offs {1:9} len {2:8}'.format(tv[0], tv[1], tv[2])) -v = mwm.read_version() -print('Format: {0}, version: {1}'.format(v['fmt'], v['date'].strftime('%Y-%m-%d %H:%M'))) -print('Header: {0}'.format(mwm.read_header())) -print('Region Info: {0}'.format(mwm.read_region_info())) -print('Metadata count: {0}'.format(len(mwm.read_metadata()))) +def dump_mwm(path): + mwm = MWM(open(path, "rb")) + mwm.read_types(os.path.join(os.path.dirname(sys.argv[0]), + "..", "..", "..", "data", "types.txt")) + print("Tags:") + tvv = sorted([(k, v[0], v[1]) for k, v in mwm.tags.items()], key=lambda x: x[1]) + for tv in tvv: + print(" {0:<8}: offs {1:9} len {2:8}".format(tv[0], tv[1], tv[2])) + v = mwm.read_version() + print("Format: {0}, version: {1}".format(v["fmt"], v["date"].strftime("%Y-%m-%d %H:%M"))) + print("Header: {0}".format(mwm.read_header())) + print("Region Info: {0}".format(mwm.read_region_info())) + print("Metadata count: {0}".format(len(mwm.read_metadata()))) -cross = mwm.read_crossmwm() -if cross: - print('Outgoing points: {0}, incoming: {1}'.format(len(cross['out']), len(cross['in']))) - print('Outgoing regions: {0}'.format(set(cross['neighbours']))) + cross = mwm.read_crossmwm() + if cross: + print("Outgoing points: {0}, incoming: {1}".format(len(cross["out"]), len(cross["in"]))) + print("Outgoing regions: {0}".format(set(cross["neighbours"]))) -# Print some random features using reservoir sampling -count = 5 -sample = [] -for i, feature in enumerate(mwm.iter_features()): - if i < count: - sample.append(feature) - elif random.randint(0, i) < count: - sample[random.randint(0, count-1)] = feature + # Print some random features using reservoir sampling + count = 5 + sample = [] + for i, feature in enumerate(mwm.iter_features()): + if i < count: + sample.append(feature) + elif random.randint(0, i) < count: + sample[random.randint(0, count-1)] = feature -print('Feature count: {0}'.format(i)) -print('Sample features:') -for feature in sample: - print(json.dumps(feature, ensure_ascii=False)) + print("Feature count: {0}".format(i)) + print("Sample features:") + for feature in sample: + print(json.dumps(feature, ensure_ascii=False)) diff --git a/tools/python/mwm/find_feature.py b/tools/python/mwm/find_feature.py index bf74b05c4b..58f9a719d2 100755 --- a/tools/python/mwm/find_feature.py +++ b/tools/python/mwm/find_feature.py @@ -1,44 +1,31 @@ -#!/usr/bin/env python import json import os.path -import sys from .mwm import MWM -if len(sys.argv) < 4: - print('Finds features in an mwm file based on a query') - print('Usage: {0} '.format(sys.argv[0])) - print('') - print('Type:') - print(' t for inside types ("t hwtag" will find all hwtags-*)') - print(' et for exact type ("et shop" won\'t find shop-chemist)') - print(' n for names, case-sensitive ("n Starbucks" for all starbucks)') - print(' m for metadata keys ("m flats" for features with flats)') - print(' id for feature id ("id 1234" for feature #1234)') - sys.exit(1) -typ = sys.argv[2].lower() -find = sys.argv[3].decode('utf-8') - -mwm = MWM(open(sys.argv[1], 'rb')) -mwm.read_header() -mwm.read_types(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'data', 'types.txt')) -for i, feature in enumerate(mwm.iter_features(metadata=True)): - found = False - if typ == 'n' and 'name' in feature['header']: - for value in feature['header']['name'].values(): - if find in value: +def find_feature(path, typ, string): + mwm = MWM(open(path, "rb")) + mwm.read_header() + mwm.read_types(os.path.join(os.path.dirname(__file__), + "..", "..", "..", "data", "types.txt")) + for i, feature in enumerate(mwm.iter_features(metadata=True)): + found = False + if typ == "n" and "name" in feature["header"]: + for value in feature["header"]["name"].values(): + if string in value: + found = True + elif typ in ("t", "et"): + for t in feature["header"]["types"]: + if t == string: + found = True + elif typ == "t" and string in t: + found = True + elif typ == "m" and "metadata" in feature: + if string in feature["metadata"]: found = True - elif typ in ('t', 'et'): - for t in feature['header']['types']: - if t == find: - found = True - elif typ == 't' and find in t: - found = True - elif typ == 'm' and 'metadata' in feature: - if find in feature['metadata']: + elif typ == "id" and i == int(string): found = True - elif typ == 'id' and i == int(find): - found = True - if found: - print(json.dumps(feature, ensure_ascii=False, sort_keys=True).encode('utf-8')) + if found: + print(json.dumps(feature, ensure_ascii=False, + sort_keys=True).encode("utf-8")) diff --git a/tools/python/mwm/ft2osm.py b/tools/python/mwm/ft2osm.py index 850f085876..fd30bc2b38 100755 --- a/tools/python/mwm/ft2osm.py +++ b/tools/python/mwm/ft2osm.py @@ -1,26 +1,12 @@ -#!/usr/bin/env python -import sys - from . import mwm -if len(sys.argv) < 3: - print('Finds an OSM object for a given feature id.') - print('Usage: {} '.format(sys.argv[0])) - sys.exit(1) -with open(sys.argv[1], 'rb') as f: - ft2osm = mwm.read_osm2ft(f, ft2osm=True) +def ft2osm(path, ftid): + with open(path, "rb") as f: + ft2osm = mwm.read_osm2ft(f, ft2osm=True) -code = 0 -type_abbr = {'n': 'node', 'w': 'way', 'r': 'relation'} -for ftid in sys.argv[2:]: + type_abbr = {"n": "node", "w": "way", "r": "relation"} ftid = int(ftid) if ftid in ft2osm: - print('https://www.openstreetmap.org/{}/{}'.format( - type_abbr[ft2osm[ftid][0]], - ft2osm[ftid][1] - )) - else: - print('Could not find osm id for feature {}'.format(ftid)) - code = 2 -sys.exit(code) + return f"https://www.openstreetmap.org/{type_abbr[ft2osm[ftid][0]]}/{ft2osm[ftid][1]}" + return None diff --git a/tools/python/mwm/mwm_feature_compare.py b/tools/python/mwm/mwm_feature_compare.py index 5587571fdd..157de85cf6 100755 --- a/tools/python/mwm/mwm_feature_compare.py +++ b/tools/python/mwm/mwm_feature_compare.py @@ -1,23 +1,19 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from __future__ import print_function - import argparse import multiprocessing import os from .mwm import MWM -OMIM_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..') +OMIM_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") def count_feature(mwm_path, feature_name): - mwm = MWM(open(mwm_path, 'rb')) + mwm = MWM(open(mwm_path, "rb")) mwm.read_header() - mwm.read_types(os.path.join(OMIM_ROOT, 'data', 'types.txt')) + mwm.read_types(os.path.join(OMIM_ROOT, "data", "types.txt")) counter = 0 for feature in mwm.iter_features(): - if feature_name in feature['header']['types']: + if feature_name in feature["header"]["types"]: counter += 1 return counter @@ -32,7 +28,7 @@ def compare_feature_num(args_tuple): p_change = float(abs(delta)) / old_feature_count * 100 if p_change > threshold: - print('In \'{0}\' number of \'{1}\' decreased by {2:.0f}% ({3} → {4})'.format( + print("In \"{0}\" number of \"{1}\" decreased by {2:.0f}% ({3} → {4})".format( os.path.basename(new_mwm), feature_name, round(p_change), old_feature_count, new_feature_count)) return False return True @@ -40,7 +36,7 @@ def compare_feature_num(args_tuple): def compare_mwm(old_mwm_path, new_mwm_path, feature_name, threshold): def valid_mwm(mwm_name): - return mwm_name.endswith('.mwm') and not mwm_name.startswith('World') + return mwm_name.endswith(".mwm") and not mwm_name.startswith("World") def generate_names_dict(path): return dict((file_name, os.path.abspath(os.path.join(path, file_name))) @@ -54,15 +50,3 @@ def compare_mwm(old_mwm_path, new_mwm_path, feature_name, threshold): pool = multiprocessing.Pool() return all(pool.imap(compare_feature_num, args)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Script to compare feature count in \'.mwm\' files') - parser.add_argument('-n', '--new', help='New mwm files path', type=str, required=True) - parser.add_argument('-o', '--old', help='Old mwm files path', type=str, required=True) - parser.add_argument('-f', '--feature', help='Feature name to count', type=str, required=True) - parser.add_argument('-t', '--threshold', help='Threshold in percent to warn', type=int, default=20) - - args = parser.parse_args() - if not compare_mwm(args.old, args.new, args.feature, args.threshold): - print('Warning: some .mwm files lost more than {}% booking hotels'.format(args.threshold)) diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 0879a816a8..cb743a03c2 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -224,14 +224,17 @@ if [ -e "$SCRIPTS_PATH/hierarchy_to_countries.py" ]; then else PYTHON_SCRIPTS_PATH="$OMIM_PATH/tools/python" fi +PYTHONPATH="${PYTHONPATH:-''}" +PYTHONPATH="${PYTHONPATH}:$PYTHON_SCRIPTS_PATH" +export PYTHONPATH ROADS_SCRIPT="$PYTHON_SCRIPTS_PATH/road_runner.py" HIERARCHY_SCRIPT="$PYTHON_SCRIPTS_PATH/post_generation/hierarchy_to_countries.py" -DESCRIPTIONS_DOWNLOADER="$PYTHON_SCRIPTS_PATH/descriptions/descriptions_downloader.py" +DESCRIPTIONS_MODULE="descriptions" LOCALADS_SCRIPT="$PYTHON_SCRIPTS_PATH/post_generation/localads_mwm_to_csv.py" UGC_FILE="${UGC_FILE:-$INTDIR/ugc_db.sqlite3}" POPULAR_PLACES_FILE="${POPULAR_PLACES_FILE:-$INTDIR/popular_places.csv}" WIKIDATA_FILE="${WIKIDATA_FILE:-$INTDIR/idToWikidata.csv}" -BOOKING_SCRIPT="$PYTHON_SCRIPTS_PATH/booking/download_hotels.py" +BOOKING_MODULE="booking" BOOKING_FILE="${BOOKING_FILE:-$INTDIR/hotels.csv}" OPENTABLE_SCRIPT="$PYTHON_SCRIPTS_PATH/opentable_restaurants.py" OPENTABLE_FILE="${OPENTABLE_FILE:-$INTDIR/restaurants.csv}" @@ -319,7 +322,7 @@ fi if [ ! -f "$BOOKING_FILE" -a -n "${BOOKING_USER-}" -a -n "${BOOKING_PASS-}" ]; then log "STATUS" "Step S1: Starting background hotels downloading" ( - $PYTHON $BOOKING_SCRIPT --user $BOOKING_USER --password $BOOKING_PASS --output "$BOOKING_FILE" --logfile="$LOG_PATH"/booking.log || true + $PYTHON36 -m $BOOKING_MODULE --user $BOOKING_USER --password $BOOKING_PASS --output "$BOOKING_FILE" --logfile="$LOG_PATH"/booking.log || true if [ -f "$BOOKING_FILE" -a "$(wc -l < "$BOOKING_FILE" || echo 0)" -gt 100 ]; then echo "Hotels have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG" else @@ -612,7 +615,7 @@ if [ "$MODE" == "descriptions" ]; then PARAMS="--wikipedia $URLS_PATH --wikidata $WIKIDATA_FILE --output_dir $WIKI_PAGES_PATH" [ -f "$POPULAR_PLACES_FILE" ] && PARAMS="$PARAMS --popularity=$POPULAR_PLACES_FILE" - $PYTHON36 $DESCRIPTIONS_DOWNLOADER $PARAMS --langs $LANGS 2>> $LOG + $PYTHON36 -m $DESCRIPTIONS_MODULE $PARAMS --langs $LANGS 2>> $LOG for file in "$TARGET"/*.mwm; do if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then