From 2385e8d9b95076acf94f72c43940a0d071269bd2 Mon Sep 17 00:00:00 2001 From: Mikhail Gorbushin Date: Thu, 24 Oct 2019 20:52:54 +0300 Subject: [PATCH] [routing] May be fix enormous memory usage in routes_builder_tool --- routing/routes_builder/routes_builder.cpp | 2 ++ routing/routes_builder/routes_builder.hpp | 1 + .../routes_builder_tool.cpp | 10 ++++-- .../routes_builder_tool/utils.cpp | 32 ++++++++++--------- .../routes_builder_tool/utils.hpp | 9 +++--- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/routing/routes_builder/routes_builder.cpp b/routing/routes_builder/routes_builder.cpp index 798a8b1d3a..bea7f737f9 100644 --- a/routing/routes_builder/routes_builder.cpp +++ b/routing/routes_builder/routes_builder.cpp @@ -291,6 +291,8 @@ RoutesBuilder::Processor::operator()(Params const & params) RouterResultCode resultCode; routing::Route route("" /* router */, 0 /* routeId */); + m_delegate->SetTimeout(params.m_timeoutSeconds); + CHECK(m_dataSource, ()); resultCode = m_router->CalculateRoute(params.m_checkpoints, diff --git a/routing/routes_builder/routes_builder.hpp b/routing/routes_builder/routes_builder.hpp index fe63464f4c..c0754bd251 100644 --- a/routing/routes_builder/routes_builder.hpp +++ b/routing/routes_builder/routes_builder.hpp @@ -60,6 +60,7 @@ public: VehicleType m_type = VehicleType::Car; Checkpoints m_checkpoints; + uint32_t m_timeoutSeconds = 0; }; struct Route diff --git a/routing/routes_builder/routes_builder_tool/routes_builder_tool.cpp b/routing/routes_builder/routes_builder_tool/routes_builder_tool.cpp index 3c345b4d17..3a281ea1ba 100644 --- a/routing/routes_builder/routes_builder_tool/routes_builder_tool.cpp +++ b/routing/routes_builder/routes_builder_tool/routes_builder_tool.cpp @@ -32,7 +32,10 @@ DEFINE_string(resources_path, "", "Resources path."); DEFINE_string(api_name, "", "Api name, current options: mapbox,google"); DEFINE_string(api_token, "", "Token for chosen api."); -DEFINE_int64(start_from, 0, "The line number from which the tool should start reading."); +DEFINE_uint64(start_from, 0, "The line number from which the tool should start reading."); + +DEFINE_int32(timeout, 10 * 60, "Timeout in seconds for each route building. " + "0 means without timeout (default: 10 minutes)."); using namespace routing; using namespace routes_builder; @@ -61,6 +64,8 @@ int Main(int argc, char ** argv) google::SetUsageMessage("This tool provides routes building for them further analyze."); google::ParseCommandLineFlags(&argc, &argv, true); + CHECK_GREATER_OR_EQUAL(FLAGS_timeout, 0, ("Timeout should be greater than zero.")); + CHECK(!FLAGS_routes_file.empty(), ("\n\n\t--routes_file is required.", "\n\nType --help for usage.")); @@ -86,9 +91,8 @@ int Main(int argc, char ** argv) else CHECK_EQUAL(Platform::MkDir(FLAGS_dump_path), Platform::EError::ERR_OK,()); - std::vector results; if (IsLocalBuild()) - results = BuildRoutes(FLAGS_routes_file, FLAGS_dump_path, FLAGS_start_from, FLAGS_threads); + BuildRoutes(FLAGS_routes_file, FLAGS_dump_path, FLAGS_start_from, FLAGS_threads, FLAGS_timeout); if (IsApiBuild()) { diff --git a/routing/routes_builder/routes_builder_tool/utils.cpp b/routing/routes_builder/routes_builder_tool/utils.cpp index 400b823ab2..b3543f0744 100644 --- a/routing/routes_builder/routes_builder_tool/utils.cpp +++ b/routing/routes_builder/routes_builder_tool/utils.cpp @@ -49,17 +49,18 @@ namespace routing { namespace routes_builder { -std::vector BuildRoutes(std::string const & routesPath, - std::string const & dumpPath, - int64_t startFrom, - uint64_t threadsNumber) +void BuildRoutes(std::string const & routesPath, + std::string const & dumpPath, + uint64_t startFrom, + uint64_t threadsNumber, + uint32_t timeoutPerRouteSeconds) { CHECK(Platform::IsFileExistsByFullPath(routesPath), ("Can not find file:", routesPath)); + CHECK(!dumpPath.empty(), ("Empty dumpPath.")); std::ifstream input(routesPath); CHECK(input.good(), ("Error during opening:", routesPath)); - std::vector result; if (!threadsNumber) { auto const hardwareConcurrency = std::thread::hardware_concurrency(); @@ -74,6 +75,7 @@ std::vector BuildRoutes(std::string const & routesPath, { RoutesBuilder::Params params; params.m_type = VehicleType::Car; + params.m_timeoutSeconds = timeoutPerRouteSeconds; base::ScopedLogLevelChanger changer(base::LogLevel::LERROR); ms::LatLon start; @@ -101,16 +103,18 @@ std::vector BuildRoutes(std::string const & routesPath, auto & task = tasks[i]; task.wait(); - result.emplace_back(task.get()); - if (!dumpPath.empty()) - { - std::string const fullPath = - base::JoinPath(dumpPath, std::to_string(shiftIndex) + RoutesBuilder::Result::kDumpExtension); + auto const result = task.get(); + if (result.m_code == RouterResultCode::Cancelled) + LOG_FORCE(LINFO, ("Route:", i, "(", i + 1, "line of file) was building too long.")); - RoutesBuilder::Result::Dump(result.back(), fullPath); - } + std::string const fullPath = + base::JoinPath(dumpPath, std::to_string(shiftIndex) + RoutesBuilder::Result::kDumpExtension); + + RoutesBuilder::Result::Dump(result, fullPath); + + double const curPercent = + static_cast(shiftIndex + 1) / (tasks.size() + startFrom) * 100.0; - double const curPercent = static_cast(shiftIndex + 1) / tasks.size() * 100.0; if (curPercent - lastPercent > 1.0 || shiftIndex + 1 == tasks.size()) { lastPercent = curPercent; @@ -118,8 +122,6 @@ std::vector BuildRoutes(std::string const & routesPath, } } } - - return result; } boost::optional> ParseApiLine(std::ifstream & input) diff --git a/routing/routes_builder/routes_builder_tool/utils.hpp b/routing/routes_builder/routes_builder_tool/utils.hpp index 9a339d5d07..01c50168aa 100644 --- a/routing/routes_builder/routes_builder_tool/utils.hpp +++ b/routing/routes_builder/routes_builder_tool/utils.hpp @@ -13,10 +13,11 @@ namespace routing { namespace routes_builder { -std::vector BuildRoutes(std::string const & routesPath, - std::string const & dumpPath, - int64_t startFrom, - uint64_t threadsNumber); +void BuildRoutes(std::string const & routesPath, + std::string const & dumpPath, + uint64_t startFrom, + uint64_t threadsNumber, + uint32_t timeoutPerRouteSeconds); void BuildRoutesWithApi(std::unique_ptr routingApi, std::string const & routesPath,