From 165b81e2637b2f701b6877a4ceb62790f6f8a63b Mon Sep 17 00:00:00 2001 From: vng Date: Sun, 11 Apr 2021 09:11:13 +0300 Subject: [PATCH] [generator][script] Fixed diffs building. Signed-off-by: vng --- .../mwm_diff/mwm_diff_tool/CMakeLists.txt | 2 +- .../mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp | 38 +++++++++++++------ .../python/maps_generator/generator/diffs.py | 11 +++--- tools/python/maps_generator/generator/env.py | 7 ++++ .../maps_generator/generator/settings.py | 10 ++--- .../generator/stages_declaration.py | 8 ++-- .../var/etc/map_generator.ini.default | 3 +- 7 files changed, 50 insertions(+), 29 deletions(-) diff --git a/generator/mwm_diff/mwm_diff_tool/CMakeLists.txt b/generator/mwm_diff/mwm_diff_tool/CMakeLists.txt index 9b68cca840..38e7dfc4ee 100644 --- a/generator/mwm_diff/mwm_diff_tool/CMakeLists.txt +++ b/generator/mwm_diff/mwm_diff_tool/CMakeLists.txt @@ -1,4 +1,4 @@ -project(pymwm_diff) +project(mwm_diff_tool) set(SRC mwm_diff_tool.cpp) diff --git a/generator/mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp b/generator/mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp index eeca988906..e66fa61c11 100644 --- a/generator/mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp +++ b/generator/mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp @@ -7,26 +7,40 @@ int main(int argc, char ** argv) { - if (argc < 5) + auto const ShowUsage = [argv]() { std::cout << - "Usage: " << argv[0] << " make|apply olderMWMDir newerMWMDir diffDir\n" + "Usage: " << argv[0] << " make|apply olderMWMPath newerMWMPath diffPath\n" "make\n" - " Creates the diff between newer and older MWM versions at `diffDir`\n" + " Creates the diff between newer and older MWMs at `diffPath`\n" "apply\n" - " Applies the diff at `diffDir` to the mwm at `olderMWMDir` and stores result at `newerMWMDir`.\n" + " Applies the diff at `diffPath` to the mwm at `olderMWMPath` and stores result at `newerMWMPath`.\n" "WARNING: THERE IS NO MWM VALIDITY CHECK!\n"; + }; + + if (argc < 5) + { + ShowUsage(); return -1; } - char const * olderMWMDir{argv[2]}, * newerMWMDir{argv[3]}, * diffDir{argv[4]}; - if (0 == std::strcmp(argv[1], "make")) - return generator::mwm_diff::MakeDiff(olderMWMDir, newerMWMDir, diffDir); - // apply - base::Cancellable cancellable; - auto const res = generator::mwm_diff::ApplyDiff(olderMWMDir, newerMWMDir, diffDir, cancellable); - if (res == generator::mwm_diff::DiffApplicationResult::Ok) - return 0; + auto const IsEqualUsage = [argv](char const * s) { return 0 == std::strcmp(argv[1], s); }; + char const * olderMWMPath{argv[2]}, * newerMWMPath{argv[3]}, * diffPath{argv[4]}; + + if (IsEqualUsage("make")) + { + if (generator::mwm_diff::MakeDiff(olderMWMPath, newerMWMPath, diffPath)) + return 0; + } + else if (IsEqualUsage("apply")) + { + base::Cancellable cancellable; + auto const res = generator::mwm_diff::ApplyDiff(olderMWMPath, newerMWMPath, diffPath, cancellable); + if (res == generator::mwm_diff::DiffApplicationResult::Ok) + return 0; + } + else + ShowUsage(); return -1; // Failed, Cancelled. } diff --git a/tools/python/maps_generator/generator/diffs.py b/tools/python/maps_generator/generator/diffs.py index 9b9df202b0..1d9efab754 100644 --- a/tools/python/maps_generator/generator/diffs.py +++ b/tools/python/maps_generator/generator/diffs.py @@ -18,9 +18,7 @@ class Status: def calculate_diff(params): - new, old, out = params["new"], params["old"], params["out"] - - diff_size = 0 + diff_tool, new, old, out = params["tool"], params["new"], params["old"], params["out"] if not new.exists(): return Status.NO_NEW_VERSION, params @@ -32,7 +30,7 @@ def calculate_diff(params): if out.exists(): status = Status.NOTHING_TO_DO else: - res = subprocess.run("mwm_diff_tool " + old.as_posix() + " " + new.as_posix() + " " + out.as_posix()) + res = subprocess.run([diff_tool.as_posix(), "make", old, new, out]) if res.returncode != 0: return Status.INTERNAL_ERROR, params @@ -52,7 +50,6 @@ def calculate_diff(params): def mwm_diff_calculation(data_dir, logger, depth): data = list(data_dir.get_mwms())[:depth] - results = map(calculate_diff, data) for status, params in results: if Status.is_error(status): @@ -61,7 +58,8 @@ def mwm_diff_calculation(data_dir, logger, depth): class DataDir(object): - def __init__(self, mwm_name, new_version_dir, old_version_root_dir): + def __init__(self, diff_tool, mwm_name, new_version_dir, old_version_root_dir): + self.diff_tool_path = Path(diff_tool) self.mwm_name = mwm_name self.diff_name = self.mwm_name + ".mwmdiff" @@ -80,6 +78,7 @@ class DataDir(object): diff_dir = Path(self.new_version_dir, old_version_dir.name) diff_dir.mkdir(exist_ok=True) yield { + "tool": self.diff_tool_path, "new": self.new_version_path, "old": Path(old_version_dir, self.mwm_name), "out": Path(diff_dir, self.diff_name) diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 477074e9ac..b02545e99b 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -420,6 +420,7 @@ class Env: self.gen_tool = self.setup_generator_tool() if WORLD_NAME in self.countries: self.world_roads_builder_tool = self.setup_world_roads_builder_tool() + self.diff_tool = self.setup_mwm_diff_tool() logger.info(f"Build name is {self.build_name}.") logger.info(f"Build path is {self.build_path}.") @@ -531,6 +532,12 @@ class Env: logger.info(f"world_roads_builder_tool found - {world_roads_builder_tool_path}") return world_roads_builder_tool_path + @staticmethod + def setup_mwm_diff_tool() -> AnyStr: + logger.info(f"Check mwm_diff_tool. Looking for it in {settings.BUILD_PATH} ...") + mwm_diff_tool_path = find_executable(settings.BUILD_PATH, "mwm_diff_tool") + logger.info(f"mwm_diff_tool found - {mwm_diff_tool_path}") + return mwm_diff_tool_path @staticmethod def setup_osm_tools() -> Dict[AnyStr, AnyStr]: diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index d8bcdbf38e..6d861f6dda 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -97,7 +97,7 @@ NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" # Stages section: NEED_PLANET_UPDATE = False THREADS_COUNT_FEATURES_STAGE = multiprocessing.cpu_count() -DATA_ARCHIVE_DIR = USER_RESOURCE_PATH +DATA_ARCHIVE_DIR = "" DIFF_VERSION_DEPTH = 2 # Logging section: @@ -235,11 +235,11 @@ def init(default_settings_path: AnyStr): global THREADS_COUNT_FEATURES_STAGE NEED_PLANET_UPDATE = cfg.get_opt("Stages", "NEED_PLANET_UPDATE", NEED_PLANET_UPDATE) DATA_ARCHIVE_DIR = cfg.get_opt_path( - "Generator tool", "DATA_ARCHIVE_DIR", DATA_ARCHIVE_DIR - ) - DIFF_VERSION_DEPTH = cfg.get_opt( - "Generator tool", "DIFF_VERSION_DEPTH", DIFF_VERSION_DEPTH + "Stages", "DATA_ARCHIVE_DIR", DATA_ARCHIVE_DIR ) + DIFF_VERSION_DEPTH = int(cfg.get_opt( + "Stages", "DIFF_VERSION_DEPTH", DIFF_VERSION_DEPTH + )) threads_count = int( cfg.get_opt( diff --git a/tools/python/maps_generator/generator/stages_declaration.py b/tools/python/maps_generator/generator/stages_declaration.py index 35ee8ee699..a455b87352 100644 --- a/tools/python/maps_generator/generator/stages_declaration.py +++ b/tools/python/maps_generator/generator/stages_declaration.py @@ -314,13 +314,13 @@ class StageRoutingTransit(Stage): @country_stage -@production_only class StageMwmDiffs(Stage): def apply(self, env: Env, country, logger, **kwargs): data_dir = diffs.DataDir( - mwm_name=env.build_name, - new_version_dir=env.build_path, - old_version_root_dir=settings.DATA_ARCHIVE_DIR, + diff_tool = env.diff_tool, + mwm_name = f"{country}.mwm", + new_version_dir = env.paths.mwm_path, + old_version_root_dir = settings.DATA_ARCHIVE_DIR, ) diffs.mwm_diff_calculation(data_dir, logger, depth=settings.DIFF_VERSION_DEPTH) diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 4f6f169026..3af13c82d5 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -87,7 +87,8 @@ NEED_BUILD_WORLD_ROADS: false # Set to 1 to update the entire OSM planet file (as taken from "planet.openstreetmap.org") # via an osmupdate tool before the generation. Not for use with partial planet extracts. NEED_PLANET_UPDATE: 0 -# If you want to calculate diffs you need to specify where the old maps are. +# If you want to calculate diffs you need to specify where the old maps are, +# e.g. ${Main:MAIN_OUT_PATH}/2021_03_16__09_00_00/ DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH} # How many versions in the archive to use for diff calculation: DIFF_VERSION_DEPTH: 2