From 43fba6ddfa0f23490719880e5edfa31a9748c640 Mon Sep 17 00:00:00 2001 From: gmoryes Date: Sun, 15 Dec 2019 20:01:36 +0300 Subject: [PATCH] [routing] smart distribution of boosting --- .../benchmark_results.hpp | 7 ++ .../routing_quality_tool/benchmark_stat.cpp | 118 +++++++++++++++--- .../routing_quality_tool/utils.cpp | 9 +- .../routing_quality_tool/utils.hpp | 3 +- 4 files changed, 120 insertions(+), 17 deletions(-) diff --git a/routing/routing_quality/routing_quality_tool/benchmark_results.hpp b/routing/routing_quality/routing_quality_tool/benchmark_results.hpp index db54b4ee6f..96f397a6cd 100644 --- a/routing/routing_quality/routing_quality_tool/benchmark_results.hpp +++ b/routing/routing_quality/routing_quality_tool/benchmark_results.hpp @@ -25,4 +25,11 @@ private: // string representation of RouterResultCode to number of such codes. std::map m_errorCounter; }; + +struct TimeInfo +{ + TimeInfo(double oldTime, double newTime) : m_oldTime(oldTime), m_newTime(newTime) {} + double m_oldTime; + double m_newTime; +}; } // namespace routing_quality::routing_quality_tool diff --git a/routing/routing_quality/routing_quality_tool/benchmark_stat.cpp b/routing/routing_quality/routing_quality_tool/benchmark_stat.cpp index 6f8bedfe92..c83ca84698 100644 --- a/routing/routing_quality/routing_quality_tool/benchmark_stat.cpp +++ b/routing/routing_quality/routing_quality_tool/benchmark_stat.cpp @@ -10,6 +10,7 @@ #include "base/assert.hpp" #include "base/file_name_utils.hpp" #include "base/logging.hpp" +#include "base/string_utils.hpp" #include @@ -17,6 +18,7 @@ namespace { using namespace routing; using namespace routes_builder; +using namespace routing_quality::routing_quality_tool; bool IsErrorCode(RouterResultCode code) { @@ -40,6 +42,70 @@ void LogIfNotConsistent(RoutesBuilder::Result const & oldRes, RoutesBuilder::Res newRoute.m_eta, "new distance:", newRoute.m_distance, start, finish)); } } + +/// \brief Helps to compare route time building for routes group by old time building. +void FillInfoAboutBuildTimeGroupByPreviousResults(std::vector & labels, + std::vector> & bars, + std::vector && times) +{ + bars.clear(); + labels.clear(); + + std::sort(times.begin(), times.end(), [](auto const & a, auto const & b) { + return a.m_oldTime < b.m_oldTime; + }); + + size_t constexpr kSteps = 10; + size_t const step = times.size() / kSteps; + + size_t startFrom = 0; + size_t curCount = 0; + bars.resize(2); + labels.clear(); + double curSumOld = 0; + double curSumNew = 0; + for (size_t i = 0; i < times.size(); ++i) + { + if (curCount < step && i + 1 != times.size()) + { + ++curCount; + curSumOld += times[i].m_oldTime; + curSumNew += times[i].m_newTime; + continue; + } + + double const curLeft = times[startFrom].m_oldTime; + startFrom = i + 1; + double const curRight = times[i].m_oldTime; + labels.emplace_back("[" + strings::to_string_dac(curLeft, 2 /* dac */) + "s, " + + strings::to_string_dac(curRight, 2 /* dac */) + "s]\\n" + + "Routes count:\\n" + std::to_string(curCount)); + curSumOld /= curCount; + curSumNew /= curCount; + double const k = curSumNew / curSumOld; + bars[0].emplace_back(100.0); + bars[1].emplace_back(100.0 * k); + curCount = 0; + } +} + +std::vector GetBoostPercents(BenchmarkResults const & oldResults, + BenchmarkResults const & newResults) +{ + std::vector boostPercents; + for (size_t i = 0; i < oldResults.GetBuildTimes().size(); ++i) + { + auto const oldTime = oldResults.GetBuildTimes()[i]; + auto const newTime = newResults.GetBuildTimes()[i]; + if (base::AlmostEqualAbs(oldTime, newTime, 1e-2)) + continue; + + auto const diffPercent = (oldTime - newTime) / oldTime * 100.0; + boostPercents.emplace_back(diffPercent); + } + + return boostPercents; +} } // namespace namespace routing_quality::routing_quality_tool @@ -60,6 +126,11 @@ static std::string const kPythonBarBoostPercentDistr = "show_boost_distr.py"; static std::string const kPythonEtaDiff = "eta_diff.py"; // The same as above but in percents. static std::string const kPythonEtaDiffPercent = "eta_diff_percent.py"; +// Groups routes by previous time building and draws two types of bars. The first one (old mapsme) +// has the same height in all groups and the second ones' height is proportional less or more +// according to difference in average time building between old and new version. The example you can +// see here: https://github.com/mapsme/omim/pull/12401 +static std::string const kPythonSmartDistr = "show_smart_boost_distr.py"; void RunBenchmarkStat( std::vector> const & mapsmeResults, @@ -122,6 +193,8 @@ void RunBenchmarkComparison( std::vector etaDiffsPercent; std::vector etaDiffs; + std::vector times; + for (size_t i = 0; i < mapsmeResults.size(); ++i) { auto const & mapsmeResult = mapsmeResults[i].first; @@ -159,15 +232,33 @@ void RunBenchmarkComparison( etaDiffs.emplace_back(etaDiff); etaDiffsPercent.emplace_back(etaDiffPercent); + auto const oldTime = mapsmeOldResult.m_buildTimeSeconds; + auto const newTime = mapsmeResult.m_buildTimeSeconds; + auto const diffPercent = (oldTime - newTime) / oldTime * 100.0; + // Warn about routes building time degradation. + double constexpr kSlowdownPercent = -10.0; + if (diffPercent < kSlowdownPercent) + { + auto const start = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetStart()); + auto const finish = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetFinish()); + LOG(LINFO, ("oldTime:", oldTime, "newTime:", newTime, "diffPercent:", diffPercent, start, finish)); + } + benchmarkResults.PushBuildTime(mapsmeResult.m_buildTimeSeconds); benchmarkOldResults.PushBuildTime(mapsmeOldResult.m_buildTimeSeconds); + + times.emplace_back(mapsmeOldResult.m_buildTimeSeconds, mapsmeResult.m_buildTimeSeconds); } LOG(LINFO, ("Comparing", benchmarkResults.GetBuildTimes().size(), "routes.")); + auto const oldAverage = benchmarkOldResults.GetAverageBuildTime(); + auto const newAverage = benchmarkResults.GetAverageBuildTime(); + auto const averageTimeDiff = (oldAverage - newAverage) / oldAverage * 100.0; LOG(LINFO, ("Average route time building. " - "Old version:", benchmarkOldResults.GetAverageBuildTime(), - "New version:", benchmarkResults.GetAverageBuildTime())); + "Old version:", oldAverage, + "New version:", newAverage, + "(", -averageTimeDiff, "% )")); std::vector> graphics; for (auto const & results : {benchmarkOldResults, benchmarkResults}) @@ -205,18 +296,7 @@ void RunBenchmarkComparison( {"old mapsme", "new mapsme"} /* legends */, "Type of errors" /* xlabel */, "Number of errors" /* ylabel */); - std::vector boostPercents; - for (size_t i = 0; i < benchmarkOldResults.GetBuildTimes().size(); ++i) - { - auto const oldTime = benchmarkOldResults.GetBuildTimes()[i]; - auto const newTime = benchmarkResults.GetBuildTimes()[i]; - if (base::AlmostEqualAbs(oldTime, newTime, 1e-2)) - continue; - - auto const diffPercent = (oldTime - newTime) / oldTime * 100.0; - boostPercents.emplace_back(diffPercent); - } - + auto const boostPercents = GetBoostPercents(benchmarkOldResults, benchmarkResults); pythonScriptPath = base::JoinPath(dirForResults, kPythonBarBoostPercentDistr); CreatePythonScriptForDistribution(pythonScriptPath, "Boost percent" /* title */, boostPercents); @@ -227,5 +307,15 @@ void RunBenchmarkComparison( pythonScriptPath = base::JoinPath(dirForResults, kPythonEtaDiffPercent); CreatePythonScriptForDistribution(pythonScriptPath, "ETA diff percent distribution" /* title */, etaDiffsPercent); + + std::vector> bars; + FillInfoAboutBuildTimeGroupByPreviousResults(labels, bars, std::move(times)); + pythonScriptPath = base::JoinPath(dirForResults, kPythonSmartDistr); + CreatePythonBarByMap( + pythonScriptPath, labels, bars, {"old mapsme", "new mapsme"} /* legends */, + "Intervals of groups (build time in old mapsme)" /* xlabel */, + "Boost\\nRight column is so lower/higher than the left\\n how much the average build time " + "has decreased in each group)" /* ylabel */, + false /* drawPercents */); } } // namespace routing_quality::routing_quality_tool diff --git a/routing/routing_quality/routing_quality_tool/utils.cpp b/routing/routing_quality/routing_quality_tool/utils.cpp index cf2cf95eee..4bb9a1f541 100644 --- a/routing/routing_quality/routing_quality_tool/utils.cpp +++ b/routing/routing_quality/routing_quality_tool/utils.cpp @@ -332,7 +332,8 @@ void CreatePythonBarByMap(std::string const & pythonScriptPath, std::vector> const & barHeights, std::vector const & legends, std::string const & xlabel, - std::string const & ylabel) + std::string const & ylabel, + bool drawPercents) { std::ofstream python(pythonScriptPath); CHECK(python.good(), ("Can not open:", pythonScriptPath, "for writing.")); @@ -348,6 +349,10 @@ void CreatePythonBarByMap(std::string const & pythonScriptPath, else counts.back() = ']'; + std::string const formatString = drawPercents + ? "'{{:2.0f}}({:2.0f}%)'.format(height, height / summ * 100)" + : "'{:2.0f}'.format(height)"; + python << R"( import matplotlib import matplotlib.pyplot as plt @@ -381,7 +386,7 @@ def autolabel(rects, counts_ith): for rect in rects: height = rect.get_height() - ax.annotate('{}({:2.0f}%)'.format(height, height / summ * 100), + ax.annotate()" + formatString + R"(, xy=(rect.get_x() + rect.get_width() / 2, height), xytext=(0, 3), # 3 points vertical offset textcoords="offset points", diff --git a/routing/routing_quality/routing_quality_tool/utils.hpp b/routing/routing_quality/routing_quality_tool/utils.hpp index 10540b415d..8ef8899daf 100644 --- a/routing/routing_quality/routing_quality_tool/utils.hpp +++ b/routing/routing_quality/routing_quality_tool/utils.hpp @@ -187,6 +187,7 @@ void CreatePythonBarByMap(std::string const & pythonScriptPath, std::vector> const & barHeights, std::vector const & legends, std::string const & xlabel, - std::string const & ylabel); + std::string const & ylabel, + bool drawPercents = true); } // namespace routing_quality_tool } // namespace routing_quality