From 664b458998fb0f2405f68ae830c2798e027b2dcc Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 2 Oct 2020 08:37:47 +0300 Subject: [PATCH] [python][packaging] Rework data packaging - Borders package contains borders.tar.xz archive. - Added data_files package. --- tools/python/data/all/setup.py | 10 +--- tools/python/data/base.py | 7 ++- tools/python/data/borders/__init__.py | 44 +++++++++++++++ tools/python/data/borders/setup.py | 55 ++++++++++++++++++- tools/python/data/essential/setup.py | 2 + tools/python/data/fonts/setup.py | 2 + tools/python/data/styles/setup.py | 2 + tools/python/data_files/__init__.py | 43 +++++++++++++++ tools/python/data_files/setup.py | 30 ++++++++++ .../maps_generator/generator/settings.py | 18 ++++-- tools/python/maps_generator/requirements.txt | 1 + tools/python/mwm/__init__.py | 16 +++--- tools/python/mwm/requirements.txt | 1 + 13 files changed, 207 insertions(+), 24 deletions(-) create mode 100644 tools/python/data/borders/__init__.py create mode 100644 tools/python/data_files/__init__.py create mode 100644 tools/python/data_files/setup.py diff --git a/tools/python/data/all/setup.py b/tools/python/data/all/setup.py index 2acc63220e..e6b28948c6 100755 --- a/tools/python/data/all/setup.py +++ b/tools/python/data/all/setup.py @@ -5,21 +5,17 @@ import sys module_dir = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(module_dir, "..", "..")) +from data.base import get_version from data.base import setup -module_dir = os.path.abspath(os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(module_dir, "..", "..", "..", "..")) - -from pyhelpers.setup import get_version - - _V = get_version() _D = [ "omim-data-borders", "omim-data-essential", + "omim-data-files", "omim-data-fonts", "omim-data-styles", ] -setup(__file__, "all", [], ["{}=={}".format(d, _V) for d in _D]) +setup(__file__, "all", [], install_requires=["{}=={}".format(d, _V) for d in _D]) diff --git a/tools/python/data/base.py b/tools/python/data/base.py index f426910e98..ab79b254ad 100644 --- a/tools/python/data/base.py +++ b/tools/python/data/base.py @@ -42,7 +42,10 @@ def setup( source_file, suffix, relative_data_paths, + packages=None, + package_dir=None, install_requires=None, + cmdclass=None, supported_pythons=("2", "2.7", "3", "3.5", "3.6", "3.7"), ): with chdir(os.path.abspath(os.path.dirname(source_file))): @@ -53,7 +56,9 @@ def setup( author_email="dev@maps.me", description="This package contains {} data files.".format(suffix), url="https://github.com/mapsme", - packages=[], + packages=[] if packages is None else packages, + package_dir={} if package_dir is None else package_dir, + cmdclass={} if cmdclass is None else cmdclass, classifiers=["License :: OSI Approved :: Apache Software License",] + [ "Programming Language :: Python :: {}".format(supported_python) diff --git a/tools/python/data/borders/__init__.py b/tools/python/data/borders/__init__.py new file mode 100644 index 0000000000..3b3e14e759 --- /dev/null +++ b/tools/python/data/borders/__init__.py @@ -0,0 +1,44 @@ +import logging +import os +import tarfile + +from six import BytesIO + +from data_files import find_data_files + +try: + import lzma +except ImportError: + from backports import lzma + +logger = logging.getLogger(__name__) + + +def init(borders_path=None): + data_path = find_data_files("omim-data") + + if data_path is None: + logger.error("omim-data was not found.") + return False + + if borders_path is None: + borders_path = os.path.join(data_path, "borders") + + if not os.path.exists(borders_path): + tar_lzma_path = os.path.join(data_path, "borders.tar.xz") + lzma_stream = BytesIO() + with open(tar_lzma_path, mode="rb") as f: + decompressed = lzma.decompress(f.read()) + lzma_stream.write(decompressed) + + lzma_stream.seek(0) + try: + with tarfile.open(fileobj=lzma_stream, mode="r") as tar: + tar.extractall(borders_path) + except PermissionError as e: + logger.error(str(e)) + return False + + logger.info("{} was created.".format(borders_path)) + + return True diff --git a/tools/python/data/borders/setup.py b/tools/python/data/borders/setup.py index 59d4bc53f5..c0e41f3327 100755 --- a/tools/python/data/borders/setup.py +++ b/tools/python/data/borders/setup.py @@ -1,10 +1,63 @@ #!/usr/bin/env python import os import sys +import tarfile +from distutils import log +from distutils.command.build import build +from distutils.command.clean import clean + +from six import BytesIO + +try: + import lzma +except ImportError: + from backports import lzma + module_dir = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(module_dir, "..", "..")) +from data.base import DATA_PATH +from data.base import chdir +from data.base import get_version from data.base import setup -setup(__file__, "borders", ["borders", "packed_polygons.bin"]) + +TAR_LZMA_PATH = os.path.join(DATA_PATH, "borders.tar.xz") + + +class BuildCmd(build, object): + def run(self): + log.info("Creating {}".format(TAR_LZMA_PATH)) + tar_stream = BytesIO() + borders_path = os.path.join(DATA_PATH, "borders") + with chdir(borders_path): + with tarfile.open(fileobj=tar_stream, mode="w") as tar: + for f in os.listdir(borders_path): + tar.add(f) + + tar_stream.seek(0) + with lzma.open(TAR_LZMA_PATH, mode="w") as f: + f.write(tar_stream.read()) + + super(BuildCmd, self).run() + + +class CleanCmd(clean, object): + def run(self): + if os.path.exists(TAR_LZMA_PATH): + log.info("Removing {}".format(TAR_LZMA_PATH)) + os.remove(TAR_LZMA_PATH) + + super(CleanCmd, self).run() + + +setup( + __file__, + "borders", + ["borders.tar.xz", "packed_polygons.bin"], + package_dir={"borders": ""}, + packages=["borders"], + cmdclass={"build": BuildCmd, "clean": CleanCmd}, + install_requires=["omim-data-files=={}".format(get_version())] +) diff --git a/tools/python/data/essential/setup.py b/tools/python/data/essential/setup.py index 36369e73d3..264c57a519 100755 --- a/tools/python/data/essential/setup.py +++ b/tools/python/data/essential/setup.py @@ -5,6 +5,7 @@ import sys module_dir = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(module_dir, "..", "..")) +from data.base import get_version from data.base import setup setup( @@ -42,4 +43,5 @@ setup( "unicode_blocks.txt", "visibility.txt", ], + install_requires=["omim-data-files=={}".format(get_version())] ) diff --git a/tools/python/data/fonts/setup.py b/tools/python/data/fonts/setup.py index 1a21cf3a27..149f625fce 100755 --- a/tools/python/data/fonts/setup.py +++ b/tools/python/data/fonts/setup.py @@ -5,6 +5,7 @@ import sys module_dir = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(module_dir, "..", "..")) +from data.base import get_version from data.base import setup setup( @@ -20,4 +21,5 @@ setup( "06_code2000.ttf", "07_roboto_medium.ttf", ], + install_requires=["omim-data-files=={}".format(get_version())] ) diff --git a/tools/python/data/styles/setup.py b/tools/python/data/styles/setup.py index 263d7a7f48..083f161fe6 100755 --- a/tools/python/data/styles/setup.py +++ b/tools/python/data/styles/setup.py @@ -5,6 +5,7 @@ import sys module_dir = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(module_dir, "..", "..")) +from data.base import get_version from data.base import setup setup( @@ -22,4 +23,5 @@ setup( "drules_proto_vehicle_dark.bin", "drules_proto_vehicle_dark.txt", ], + install_requires=["omim-data-files=={}".format(get_version())] ) diff --git a/tools/python/data_files/__init__.py b/tools/python/data_files/__init__.py new file mode 100644 index 0000000000..db35ea02ae --- /dev/null +++ b/tools/python/data_files/__init__.py @@ -0,0 +1,43 @@ +import os +import site +import sys + + +def find_data_files_in_user_installations(directory): + possible_paths = [os.path.join(site.USER_BASE, directory),] + [ + os.path.normpath(os.path.join(p, "../../..", directory)) + for p in site.getusersitepackages() + ] + + for p in possible_paths: + if os.path.isdir(p): + return p + + return None + + +def find_data_files_in_sys_installations(directory): + possible_paths = [os.path.join(sys.prefix, directory),] + [ + os.path.normpath(os.path.join(p, "../../..", directory)) + for p in site.getsitepackages() + ] + for p in possible_paths: + if os.path.isdir(p): + return p + + return None + + +def find_data_files(directory, user_inst_first=True): + functions = [ + (int(user_inst_first), find_data_files_in_user_installations), + (int(not user_inst_first), find_data_files_in_sys_installations), + ] + + functions.sort(key=lambda k: k[0]) + for prior, func in functions: + res = func(directory) + if res is not None: + return res + + return None diff --git a/tools/python/data_files/setup.py b/tools/python/data_files/setup.py new file mode 100644 index 0000000000..e4db67f581 --- /dev/null +++ b/tools/python/data_files/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +import os +import sys + +import setuptools + +module_dir = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(module_dir, "..", "..", "..")) + +from pyhelpers.setup import chdir +from pyhelpers.setup import get_version + + +with chdir(os.path.abspath(os.path.dirname(__file__))): + supported_pythons = ("2", "2.7", "3", "3.5", "3.6", "3.7") + setuptools.setup( + name="omim-data-files", + version=str(get_version()), + author="My.com B.V. (Mail.Ru Group)", + author_email="dev@maps.me", + description="This package is a library for dealing with data files.", + url="https://github.com/mapsme", + package_dir={"data_files": ""}, + packages=["data_files",], + classifiers=["License :: OSI Approved :: Apache Software License",] + + [ + "Programming Language :: Python :: {}".format(supported_python) + for supported_python in supported_pythons + ], + ) diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index 71be1aa2ef..eee97a09d1 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -92,12 +92,7 @@ OSM_TOOLS_PATH = os.path.join(_WORK_PATH, "osmctools") # Generator tool section: NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" -_omim_data_dir = "omim-data" -USER_RESOURCE_PATH = os.path.join(sys.prefix, _omim_data_dir) -if not os.path.exists(USER_RESOURCE_PATH): - USER_RESOURCE_PATH = os.path.join(site.USER_BASE, _omim_data_dir) -if not os.path.exists(USER_RESOURCE_PATH): - USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data") +USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data") # Stages section: NEED_PLANET_UPDATE = False @@ -220,6 +215,17 @@ def init(default_settings_path: AnyStr): ) NODE_STORAGE = cfg.get_opt("Generator tool", "NODE_STORAGE", NODE_STORAGE) + if not os.path.exists(USER_RESOURCE_PATH): + from data_files import find_data_files + USER_RESOURCE_PATH = find_data_files("omim-data") + assert USER_RESOURCE_PATH is not None + + import borders + # Issue: If maps_generator is installed in your system as a system + # package and borders.init() is called first time, call borders.init() + # might return False, because you need root permission. + assert borders.init() + # Stages section: global NEED_PLANET_UPDATE NEED_PLANET_UPDATE = cfg.get_opt("Stages", "NEED_PLANET_UPDATE", NEED_PLANET_UPDATE) diff --git a/tools/python/maps_generator/requirements.txt b/tools/python/maps_generator/requirements.txt index 1081a8dd8b..88a9964beb 100644 --- a/tools/python/maps_generator/requirements.txt +++ b/tools/python/maps_generator/requirements.txt @@ -1,4 +1,5 @@ omim-data-all +omim-data-files omim-descriptions omim-post_generation filelock==3.0.10 diff --git a/tools/python/mwm/__init__.py b/tools/python/mwm/__init__.py index 451cb17a61..76782941d9 100644 --- a/tools/python/mwm/__init__.py +++ b/tools/python/mwm/__init__.py @@ -1,15 +1,13 @@ import os -import site -import sys -_omim_data_dir = "omim-data" -resource_path = os.path.join(sys.prefix, _omim_data_dir) +resource_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "data", +) if not os.path.exists(resource_path): - resource_path = os.path.join(site.USER_BASE, _omim_data_dir) -if not os.path.exists(resource_path): - resource_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "data", - ) + from data_files import find_data_files + resource_path = find_data_files("omim-data") + assert resource_path is not None + from mwm.feature_types import init as _init diff --git a/tools/python/mwm/requirements.txt b/tools/python/mwm/requirements.txt index 61a1cadb31..692573cbd3 100644 --- a/tools/python/mwm/requirements.txt +++ b/tools/python/mwm/requirements.txt @@ -1,2 +1,3 @@ omim-data-essential +omim-data-files omim-pygen \ No newline at end of file