[generator] download_file() now allows URLs with "file" protocol

download_file(url) function now allows URLs beginning with "file://" and pointing to a local path.

Signed-off-by: Maciej Sikorski <maciejosikorski@gmail.com>
This commit is contained in:
Maciej Sikorski 2022-09-09 17:54:44 +02:00
parent eca232021c
commit 727b36d069
2 changed files with 25 additions and 0 deletions

View file

@ -13,6 +13,7 @@ from typing import Optional
from urllib.parse import unquote
from urllib.parse import urljoin
from urllib.parse import urlparse
from urllib.request import url2pathname
import requests
from bs4 import BeautifulSoup
@ -24,6 +25,19 @@ from maps_generator.utils.md5 import md5_ext
logger = logging.getLogger("maps_generator")
def is_file_uri(url: AnyStr) -> bool:
return urlparse(url).scheme == "file"
def file_uri_to_path(url : AnyStr) -> AnyStr:
file_uri = urlparse(url)
file_path = file_uri.path
#URI is somehting like "file://~/..."
if file_uri.netloc == '~':
file_path = f'~{file_uri.path}'
return os.path.expanduser(file_path)
def is_executable(fpath: AnyStr) -> bool:
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
@ -48,6 +62,16 @@ def download_file(url: AnyStr, name: AnyStr, download_if_exists: bool = True):
logger.info(f"File {name} already exists.")
return
if is_file_uri(url):
#url uses 'file://' scheme
try:
copy_overwrite(file_uri_to_path(url), name)
except FileNotFoundError as e:
raise e
logger.info(f"File {name} was copied from {url}.")
return
tmp_name = f"{name}__"
os.makedirs(os.path.dirname(tmp_name), exist_ok=True)
with requests.Session() as session:

View file

@ -46,6 +46,7 @@ DIFF_VERSION_DEPTH: 2
[External]
# Note: If you want to set a directory name you have to add "/" to the end of url.
# In each field where you need to specify a URL, you can specify the path to the file system using file:///path/to/file
# The url to the planet file.
# PLANET_URL: