From 6ebf4621b16002d3c2238482bf7cf711138a6ca3 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Thu, 20 Jun 2019 01:49:52 +0300 Subject: [PATCH] [python][generator] Deleted popen curl. --- .../maps_generator/generator/basic_stages.py | 14 ++++---------- tools/python/maps_generator/maps_generator.py | 8 +++----- tools/python/maps_generator/utils/file.py | 13 ++++++++----- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/tools/python/maps_generator/generator/basic_stages.py b/tools/python/maps_generator/generator/basic_stages.py index c938aa5f73..e320966539 100644 --- a/tools/python/maps_generator/generator/basic_stages.py +++ b/tools/python/maps_generator/generator/basic_stages.py @@ -6,16 +6,11 @@ from ..utils.md5 import write_md5sum, md5 from . import settings from .gen_tool import run_gen_tool from .osmtools import osmconvert, osmupdate -from .exceptions import wait_and_raise_if_fail -def download_planet(planet, output=subprocess.DEVNULL, - error=subprocess.DEVNULL): - p = download_file(settings.PLANET_URL, planet, output=output, error=error) - m = download_file(settings.PLANET_MD5_URL, md5(planet), output=output, - error=error) - wait_and_raise_if_fail(p) - wait_and_raise_if_fail(m) +def download_planet(planet): + download_file(settings.PLANET_URL, planet) + download_file(settings.PLANET_MD5_URL, md5(planet)) def convert_planet(tool, in_planet, out_planet, output=subprocess.DEVNULL, @@ -26,8 +21,7 @@ def convert_planet(tool, in_planet, out_planet, output=subprocess.DEVNULL, def stage_download_and_convert_planet(env, **kwargs): if not is_verified(settings.PLANET_PBF): - download_planet(settings.PLANET_PBF, output=env.get_subprocess_out(), - error=env.get_subprocess_out()) + download_planet(settings.PLANET_PBF) convert_planet(env[settings.OSM_TOOL_CONVERT], settings.PLANET_PBF, settings.PLANET_O5M, diff --git a/tools/python/maps_generator/maps_generator.py b/tools/python/maps_generator/maps_generator.py index 7cd2fcbd9b..6e8f006163 100644 --- a/tools/python/maps_generator/maps_generator.py +++ b/tools/python/maps_generator/maps_generator.py @@ -23,8 +23,7 @@ from .generator import settings from .generator.decorators import stage, country_stage, country_stage_log from .generator.env import (planet_lock_file, build_lock_file, WORLD_COASTS_NAME, WORLD_NAME, WORLDS_NAMES) -from .generator.exceptions import (ContinueError, BadExitStatusError, - wait_and_raise_if_fail) +from .generator.exceptions import ContinueError, BadExitStatusError from .generator.gen_tool import run_gen_tool from .generator.statistics import make_stats, get_stages_info from .utils.file import is_verified, download_file, make_tarfile @@ -33,9 +32,8 @@ logger = logging.getLogger("maps_generator") def download_external(url_to_path: dict): - ps = [download_file(k, v) for k, v in url_to_path.items()] - for p in ps: - wait_and_raise_if_fail(p) + for k, v in url_to_path.items(): + download_file(k, v) @stage diff --git a/tools/python/maps_generator/utils/file.py b/tools/python/maps_generator/utils/file.py index fe7575af20..42a80142d4 100644 --- a/tools/python/maps_generator/utils/file.py +++ b/tools/python/maps_generator/utils/file.py @@ -1,13 +1,16 @@ import errno import functools import glob +import logging import os import shutil -import subprocess import tarfile +import urllib.request from .md5 import md5, check_md5 +logger = logging.getLogger("maps_generator") + def is_executable(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) @@ -27,10 +30,10 @@ def find_executable(path, exe=None): raise FileNotFoundError(f"{exe} not found in {path}") -def download_file(url, name, output=subprocess.DEVNULL, - error=subprocess.DEVNULL): - return subprocess.Popen(["curl", "-s", "-L", "-o" + name, url], - stdout=output, stderr=error) +def download_file(url, name): + logger.info(f"Trying to download {name} from {url}.") + urllib.request.urlretrieve(url, name) + logger.info(f"File {name} was downloaded from {url}.") def is_exists_file_and_md5(name):