diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 3abdee9e5d..df6133f20e 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -434,7 +434,7 @@ int GeneratorToolMain(int argc, char ** argv) auto const tmpPath = base::JoinPath(genInfo.m_intermediateDir, "tmp"); auto const datFiles = platform_helpers::GetFullDataTmpFilePaths(tmpPath); WikiUrlDumper wikiUrlDumper(FLAGS_dump_wikipedia_urls, datFiles); - wikiUrlDumper.Dump(); + wikiUrlDumper.Dump(threadsCount); } // Enumerate over all dat files that were created. diff --git a/generator/wiki_url_dumper.cpp b/generator/wiki_url_dumper.cpp index c0e67e21c6..4c5c55dab5 100644 --- a/generator/wiki_url_dumper.cpp +++ b/generator/wiki_url_dumper.cpp @@ -6,22 +6,49 @@ #include "indexer/feature_processor.hpp" #include "indexer/ftypes_matcher.hpp" +#include "base/assert.hpp" + #include #include +#include +#include + +#include "3party/ThreadPool/ThreadPool.h" namespace generator { WikiUrlDumper::WikiUrlDumper(std::string const & path, std::vector const & dataFiles) : m_path(path), m_dataFiles(dataFiles) {} -void WikiUrlDumper::Dump() const +void WikiUrlDumper::Dump(size_t cpuCount) const { + CHECK_GREATER(cpuCount, 0, ()); + + ThreadPool threadPool(cpuCount); + std::vector> futures; + futures.reserve(m_dataFiles.size()); + + auto const fn = [](std::string const & filename) { + std::stringstream stringStream; + DumpOne(filename, stringStream); + return stringStream.str(); + }; + + for (auto const & path : m_dataFiles) + { + auto result = threadPool.enqueue(fn, path); + futures.emplace_back(std::move(result)); + } + std::ofstream stream; stream.exceptions(std::fstream::failbit | std::fstream::badbit); stream.open(m_path); stream << "MwmPath\tosmId\twikipediaUrl\n"; - for (auto const & path : m_dataFiles) - DumpOne(path, stream); + for (auto & f : futures) + { + auto lines = f.get(); + stream << lines; + } } // static diff --git a/generator/wiki_url_dumper.hpp b/generator/wiki_url_dumper.hpp index cc0abbc21a..df924edf3d 100644 --- a/generator/wiki_url_dumper.hpp +++ b/generator/wiki_url_dumper.hpp @@ -13,7 +13,7 @@ public: static void DumpOne(std::string const & path, std::ostream & stream); - void Dump() const; + void Dump(size_t cpuCount) const; private: std::string m_path;