[python][packaging] Rework data packaging

- Borders package contains borders.tar.xz archive.
- Added data_files package.

tools/python/data/all/setup.py

@@ -5,21 +5,17 @@ import sys
 module_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(module_dir, "..", ".."))
 
+from data.base import get_version
 from data.base import setup
 
-module_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.insert(0, os.path.join(module_dir, "..", "..", "..", ".."))
-
-from pyhelpers.setup import get_version
-
-
 _V = get_version()
 
 _D = [
     "omim-data-borders",
     "omim-data-essential",
+    "omim-data-files",
     "omim-data-fonts",
     "omim-data-styles",
 ]
 
-setup(__file__, "all", [], ["{}=={}".format(d, _V) for d in _D])
+setup(__file__, "all", [], install_requires=["{}=={}".format(d, _V) for d in _D])

tools/python/data/base.py

@@ -42,7 +42,10 @@ def setup(
     source_file,
     suffix,
     relative_data_paths,
+    packages=None,
+    package_dir=None,
     install_requires=None,
+    cmdclass=None,
     supported_pythons=("2", "2.7", "3", "3.5", "3.6", "3.7"),
 ):
     with chdir(os.path.abspath(os.path.dirname(source_file))):
@@ -53,7 +56,9 @@ def setup(
             author_email="dev@maps.me",
             description="This package contains {} data files.".format(suffix),
             url="https://github.com/mapsme",
-            packages=[],
+            packages=[] if packages is None else packages,
+            package_dir={} if package_dir is None else package_dir,
+            cmdclass={} if cmdclass is None else cmdclass,
             classifiers=["License :: OSI Approved :: Apache Software License",]
             + [
                 "Programming Language :: Python :: {}".format(supported_python)

tools/python/data/borders/__init__.py

@@ -0,0 +1,44 @@
+import logging
+import os
+import tarfile
+
+from six import BytesIO
+
+from data_files import find_data_files
+
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
+
+logger = logging.getLogger(__name__)
+
+
+def init(borders_path=None):
+    data_path = find_data_files("omim-data")
+
+    if data_path is None:
+        logger.error("omim-data was not found.")
+        return False
+
+    if borders_path is None:
+        borders_path = os.path.join(data_path, "borders")
+
+    if not os.path.exists(borders_path):
+        tar_lzma_path = os.path.join(data_path, "borders.tar.xz")
+        lzma_stream = BytesIO()
+        with open(tar_lzma_path, mode="rb") as f:
+            decompressed = lzma.decompress(f.read())
+            lzma_stream.write(decompressed)
+
+        lzma_stream.seek(0)
+        try:
+            with tarfile.open(fileobj=lzma_stream, mode="r") as tar:
+                tar.extractall(borders_path)
+        except PermissionError as e:
+            logger.error(str(e))
+            return False
+
+        logger.info("{} was created.".format(borders_path))
+
+    return True

tools/python/data/borders/setup.py

@@ -1,10 +1,63 @@
 #!/usr/bin/env python
 import os
 import sys
+import tarfile
+from distutils import log
+from distutils.command.build import build
+from distutils.command.clean import clean
+
+from six import BytesIO
+
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
+
 
 module_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(module_dir, "..", ".."))
 
+from data.base import DATA_PATH
+from data.base import chdir
+from data.base import get_version
 from data.base import setup
 
-setup(__file__, "borders", ["borders", "packed_polygons.bin"])
+
+TAR_LZMA_PATH = os.path.join(DATA_PATH, "borders.tar.xz")
+
+
+class BuildCmd(build, object):
+    def run(self):
+        log.info("Creating {}".format(TAR_LZMA_PATH))
+        tar_stream = BytesIO()
+        borders_path = os.path.join(DATA_PATH, "borders")
+        with chdir(borders_path):
+            with tarfile.open(fileobj=tar_stream, mode="w") as tar:
+                for f in os.listdir(borders_path):
+                    tar.add(f)
+
+        tar_stream.seek(0)
+        with lzma.open(TAR_LZMA_PATH, mode="w") as f:
+            f.write(tar_stream.read())
+
+        super(BuildCmd, self).run()
+
+
+class CleanCmd(clean, object):
+    def run(self):
+        if os.path.exists(TAR_LZMA_PATH):
+            log.info("Removing {}".format(TAR_LZMA_PATH))
+            os.remove(TAR_LZMA_PATH)
+
+        super(CleanCmd, self).run()
+
+
+setup(
+    __file__,
+    "borders",
+    ["borders.tar.xz", "packed_polygons.bin"],
+    package_dir={"borders": ""},
+    packages=["borders"],
+    cmdclass={"build": BuildCmd, "clean": CleanCmd},
+    install_requires=["omim-data-files=={}".format(get_version())]
+)

tools/python/data/essential/setup.py

@@ -5,6 +5,7 @@ import sys
 module_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(module_dir, "..", ".."))
 
+from data.base import get_version
 from data.base import setup
 
 setup(
@@ -42,4 +43,5 @@ setup(
         "unicode_blocks.txt",
         "visibility.txt",
     ],
+    install_requires=["omim-data-files=={}".format(get_version())]
 )

tools/python/data/fonts/setup.py

@@ -5,6 +5,7 @@ import sys
 module_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(module_dir, "..", ".."))
 
+from data.base import get_version
 from data.base import setup
 
 setup(
@@ -20,4 +21,5 @@ setup(
         "06_code2000.ttf",
         "07_roboto_medium.ttf",
     ],
+    install_requires=["omim-data-files=={}".format(get_version())]
 )

tools/python/data/styles/setup.py

@@ -5,6 +5,7 @@ import sys
 module_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(module_dir, "..", ".."))
 
+from data.base import get_version
 from data.base import setup
 
 setup(
@@ -22,4 +23,5 @@ setup(
         "drules_proto_vehicle_dark.bin",
         "drules_proto_vehicle_dark.txt",
     ],
+    install_requires=["omim-data-files=={}".format(get_version())]
 )

tools/python/data_files/__init__.py

@@ -0,0 +1,43 @@
+import os
+import site
+import sys
+
+
+def find_data_files_in_user_installations(directory):
+    possible_paths = [os.path.join(site.USER_BASE, directory),] + [
+        os.path.normpath(os.path.join(p, "../../..", directory))
+        for p in site.getusersitepackages()
+    ]
+
+    for p in possible_paths:
+        if os.path.isdir(p):
+            return p
+
+    return None
+
+
+def find_data_files_in_sys_installations(directory):
+    possible_paths = [os.path.join(sys.prefix, directory),] + [
+        os.path.normpath(os.path.join(p, "../../..", directory))
+        for p in site.getsitepackages()
+    ]
+    for p in possible_paths:
+        if os.path.isdir(p):
+            return p
+
+    return None
+
+
+def find_data_files(directory, user_inst_first=True):
+    functions = [
+        (int(user_inst_first), find_data_files_in_user_installations),
+        (int(not user_inst_first), find_data_files_in_sys_installations),
+    ]
+
+    functions.sort(key=lambda k: k[0])
+    for prior, func in functions:
+        res = func(directory)
+        if res is not None:
+            return res
+
+    return None

tools/python/data_files/setup.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import os
+import sys
+
+import setuptools
+
+module_dir = os.path.abspath(os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(module_dir, "..", "..", ".."))
+
+from pyhelpers.setup import chdir
+from pyhelpers.setup import get_version
+
+
+with chdir(os.path.abspath(os.path.dirname(__file__))):
+    supported_pythons = ("2", "2.7", "3", "3.5", "3.6", "3.7")
+    setuptools.setup(
+        name="omim-data-files",
+        version=str(get_version()),
+        author="My.com B.V. (Mail.Ru Group)",
+        author_email="dev@maps.me",
+        description="This package is a library for dealing with data files.",
+        url="https://github.com/mapsme",
+        package_dir={"data_files": ""},
+        packages=["data_files",],
+        classifiers=["License :: OSI Approved :: Apache Software License",]
+        + [
+            "Programming Language :: Python :: {}".format(supported_python)
+            for supported_python in supported_pythons
+        ],
+    )

tools/python/maps_generator/generator/settings.py

@@ -92,12 +92,7 @@ OSM_TOOLS_PATH = os.path.join(_WORK_PATH, "osmctools")
 # Generator tool section:
 NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map"
 
-_omim_data_dir = "omim-data"
-USER_RESOURCE_PATH = os.path.join(sys.prefix, _omim_data_dir)
-if not os.path.exists(USER_RESOURCE_PATH):
-    USER_RESOURCE_PATH = os.path.join(site.USER_BASE, _omim_data_dir)
-if not os.path.exists(USER_RESOURCE_PATH):
-    USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data")
+USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data")
 
 # Stages section:
 NEED_PLANET_UPDATE = False
@@ -220,6 +215,17 @@ def init(default_settings_path: AnyStr):
     )
     NODE_STORAGE = cfg.get_opt("Generator tool", "NODE_STORAGE", NODE_STORAGE)
 
+    if not os.path.exists(USER_RESOURCE_PATH):
+        from data_files import find_data_files
+        USER_RESOURCE_PATH = find_data_files("omim-data")
+        assert USER_RESOURCE_PATH is not None
+
+        import borders
+        # Issue: If maps_generator is installed in your system as a system
+        # package and borders.init() is called first time, call borders.init()
+        # might return False, because you need root permission.
+        assert borders.init()
+
     # Stages section:
     global NEED_PLANET_UPDATE
     NEED_PLANET_UPDATE = cfg.get_opt("Stages", "NEED_PLANET_UPDATE", NEED_PLANET_UPDATE)

tools/python/maps_generator/requirements.txt

@@ -1,4 +1,5 @@
 omim-data-all
+omim-data-files
 omim-descriptions
 omim-post_generation
 filelock==3.0.10

tools/python/mwm/__init__.py

@@ -1,15 +1,13 @@
 import os
-import site
-import sys
 
-_omim_data_dir = "omim-data"
-resource_path = os.path.join(sys.prefix, _omim_data_dir)
+resource_path = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "data",
+)
 if not os.path.exists(resource_path):
-    resource_path = os.path.join(site.USER_BASE, _omim_data_dir)
-if not os.path.exists(resource_path):
-    resource_path = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "data",
-    )
+    from data_files import find_data_files
+    resource_path = find_data_files("omim-data")
+    assert resource_path is not None
+
 
 from mwm.feature_types import init as _init
 

tools/python/mwm/requirements.txt

@@ -1,2 +1,3 @@
 omim-data-essential
+omim-data-files
 omim-pygen