diff --git a/base/stl_helpers.hpp b/base/stl_helpers.hpp index d21cc6b70c..b3f020b851 100644 --- a/base/stl_helpers.hpp +++ b/base/stl_helpers.hpp @@ -142,6 +142,12 @@ bool AnyOf(Cont && c, Fn && fn) return std::any_of(c.cbegin(), c.cend(), std::forward(fn)); } +template +decltype(auto) Transform(Cont && c, OutIt && it, Fn && fn) +{ + return std::transform(std::cbegin(c), std::cend(c), std::forward(it), std::forward(fn)); +} + template decltype(auto) FindIf(Cont && c, Fn && fn) { diff --git a/generator/complex_generator/complex_generator.cpp b/generator/complex_generator/complex_generator.cpp index 597130b11b..45351c7689 100644 --- a/generator/complex_generator/complex_generator.cpp +++ b/generator/complex_generator/complex_generator.cpp @@ -106,7 +106,7 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) { else { finalProcessor->SetPrintFunction([](auto const & entry) { - return generator::popularity::HierarchyEntryToCsvString(entry); + return generator::hierarchy::HierarchyEntryToCsvString(entry); }); } rawGenerator.GenerateCustom(translator, finalProcessor); diff --git a/generator/final_processor_intermediate_mwm.cpp b/generator/final_processor_intermediate_mwm.cpp index 14aa59e5bf..772197c3ad 100644 --- a/generator/final_processor_intermediate_mwm.cpp +++ b/generator/final_processor_intermediate_mwm.cpp @@ -527,7 +527,7 @@ void ComplexFinalProcessor::SetMwmAndFt2OsmPath(std::string const & mwmPath, m_osm2ftPath = osm2ftPath; } -void ComplexFinalProcessor::SetPrintFunction(hierarchy::PrintFunction const & printFunction) +void ComplexFinalProcessor::SetPrintFunction(hierarchy::PrintFn const & printFunction) { m_printFunction = printFunction; } @@ -553,16 +553,16 @@ void ComplexFinalProcessor::Process() strings::ReplaceLast(countryName, DATA_FILE_EXTENSION_TMP, ""); hierarchy::HierarchyBuilder builder(base::JoinPath(m_mwmTmpPath, filename)); - builder.SetGetMainTypeFunction(popularity::GetMainType); - builder.SetGetNameFunction(popularity::GetName); + builder.SetGetMainTypeFunction(hierarchy::GetMainType); + builder.SetGetNameFunction(hierarchy::GetName); auto nodes = builder.Build(); auto const enricher = CreateEnricher(countryName); hierarchy::HierarchyLinesBuilder linesBuilder(std::move(nodes)); linesBuilder.SetHierarchyLineEnricher(enricher); linesBuilder.SetCountryName(countryName); - linesBuilder.SetGetMainTypeFunction(popularity::GetMainType); - linesBuilder.SetGetNameFunction(popularity::GetName); + linesBuilder.SetGetMainTypeFunction(hierarchy::GetMainType); + linesBuilder.SetGetNameFunction(hierarchy::GetName); return linesBuilder.GetHierarchyLines(); }); futures.emplace_back(std::move(future)); diff --git a/generator/final_processor_intermediate_mwm.hpp b/generator/final_processor_intermediate_mwm.hpp index f1e7be658c..cd7a6467fd 100644 --- a/generator/final_processor_intermediate_mwm.hpp +++ b/generator/final_processor_intermediate_mwm.hpp @@ -134,7 +134,7 @@ public: size_t threadsCount); void SetMwmAndFt2OsmPath(std::string const & mwmPath, std::string const & osm2ftPath); - void SetPrintFunction(hierarchy::PrintFunction const & printFunction); + void SetPrintFunction(hierarchy::PrintFn const & printFunction); // FinalProcessorIntermediateMwmInterface overrides: void Process() override; @@ -144,7 +144,7 @@ private: std::string const & countryName) const; void WriteLines(std::vector const & lines); - hierarchy::PrintFunction m_printFunction = hierarchy::PrintDefault; + hierarchy::PrintFn m_printFunction = hierarchy::PrintDefault; std::string m_mwmTmpPath; std::string m_outFilename; std::string m_mwmPath; diff --git a/generator/generator_tests/CMakeLists.txt b/generator/generator_tests/CMakeLists.txt index 65c75c82a2..9c198cedf2 100644 --- a/generator/generator_tests/CMakeLists.txt +++ b/generator/generator_tests/CMakeLists.txt @@ -21,6 +21,7 @@ set( feature_merger_test.cpp filter_elements_tests.cpp gen_mwm_info_tests.cpp + hierarchy_entry_tests.cpp intermediate_data_test.cpp maxspeeds_tests.cpp merge_collectors_tests.cpp diff --git a/generator/generator_tests/hierarchy_entry_tests.cpp b/generator/generator_tests/hierarchy_entry_tests.cpp new file mode 100644 index 0000000000..f30cf4dfaf --- /dev/null +++ b/generator/generator_tests/hierarchy_entry_tests.cpp @@ -0,0 +1,150 @@ +#include "testing/testing.hpp" + +#include "generator/composite_id.hpp" +#include "generator/generator_tests_support/test_with_classificator.hpp" +#include "generator/hierarchy_entry.hpp" + +#include "indexer/complex/tree_node.hpp" + +#include "base/geo_object_id.hpp" + +#include "platform/platform_tests_support/scoped_file.hpp" + +using generator::tests_support::TestWithClassificator; +using platform::tests_support::ScopedFile; + +namespace +{ +std::string const kCsv1 = + "13835058055284963881 9223372037111861697;" + ";" + "1;" + "37.5303271;" + "67.3684086;" + "amenity-university;" + "Lomonosov Moscow State Univesity;" + "Russia_Moscow\n" + + "9223372036879747192 9223372036879747192;" + "13835058055284963881 9223372037111861697;" + "2;" + "37.5272372;" + "67.3775872;" + "leisure-garden;" + "Ботанический сад МГУ;" + "Russia_Moscow\n" + + "9223372036938640141 9223372036938640141;" + "9223372036879747192 9223372036879747192;" + "3;" + "37.5274156;" + "67.3758813;" + "amenity-university;" + "Отдел флоры;" + "Russia_Moscow\n" + + "9223372036964008573 9223372036964008573;" + "9223372036879747192 9223372036879747192;" + "3;" + "37.5279467;" + "67.3756452;" + "amenity-university;" + "Дендрарий Ботанического сада МГУ;" + "Russia_Moscow\n" + + "4611686019330739245 4611686019330739245;" + "13835058055284963881 9223372037111861697;" + "2;" + "37.5357492;" + "67.3735142;" + "historic-memorial;" + "Александр Иванович Герцен;" + "Russia_Moscow\n" + + "4611686019330739269 4611686019330739269;" + "13835058055284963881 9223372037111861697;" + "2;" + "37.5351269;" + "67.3741606;" + "historic-memorial;" + "Николай Гаврилович Чернышевский;" + "Russia_Moscow\n" + + "4611686019330739276 4611686019330739276;" + "13835058055284963881 9223372037111861697;" + "2;" + "37.5345234;" + "67.3723206;" + "historic-memorial;" + "Николай Егорович Жуковский;" + "Russia_Moscow"; + +generator::CompositeId MakeId(unsigned long long f, unsigned long long s) +{ + return generator::CompositeId(base::GeoObjectId(static_cast(f)), + base::GeoObjectId(static_cast(s))); +} + +UNIT_CLASS_TEST(TestWithClassificator, Complex_HierarchyEntryCsv) +{ + generator::HierarchyEntry e; + e.m_id = MakeId(4611686018725364866ull, 4611686018725364866ull); + e.m_parentId = MakeId(13835058055283414237ull, 9223372037374119493ull); + e.m_depth = 2; + e.m_center.x = 37.6262604; + e.m_center.y = 67.6098812; + e.m_type = classif().GetTypeByPath({"amenity", "restaurant"}); + e.m_name = "Rybatskoye podvor'ye"; + e.m_countryName = "Russia_Moscow"; + + auto const row = generator::hierarchy::HierarchyEntryToCsvRow(e); + TEST_EQUAL(row.size(), 8, ()); + TEST_EQUAL(row[0], "4611686018725364866 4611686018725364866", ()); + TEST_EQUAL(row[1], "13835058055283414237 9223372037374119493", ()); + TEST_EQUAL(row[2], "2", ()); + TEST_EQUAL(row[3], "37.6262604", ()); + TEST_EQUAL(row[4], "67.6098812", ()); + TEST_EQUAL(row[5], "amenity-restaurant", ()); + TEST_EQUAL(row[6], "Rybatskoye podvor'ye", ()); + TEST_EQUAL(row[7], "Russia_Moscow", ()); + + auto const res = generator::hierarchy::HierarchyEntryFromCsvRow(row); + TEST_EQUAL(e, res, ()); +} + +UNIT_CLASS_TEST(TestWithClassificator, Complex_LoadHierachy) +{ + auto const filename = "test.csv"; + ScopedFile sf(filename, kCsv1); + auto const forest = generator::hierarchy::LoadHierachy(sf.GetFullPath()); + TEST_EQUAL(forest.size(), 1, ()); + auto const & tree = *forest.begin(); + LOG(LINFO, (tree)); + + TEST_EQUAL(tree_node::Size(tree), 7, ()); + auto node = tree_node::FindIf(tree, [](auto const & e) { + return e.m_id == MakeId(13835058055284963881ull, 9223372037111861697ull); + }); + TEST(node, ()); + TEST(!node->HasParent(), ()); + TEST_EQUAL(node->GetChildren().size(), 4, ()); + + node = tree_node::FindIf(tree, [](auto const & e) { + return e.m_id == MakeId(9223372036879747192ull, 9223372036879747192ull); + }); + TEST(node, ()); + TEST(node->HasParent(), ()); + TEST_EQUAL(node->GetParent()->GetData().m_id, + MakeId(13835058055284963881ull, 9223372037111861697ull), ()); + TEST_EQUAL(node->GetChildren().size(), 2, ()); + + node = tree_node::FindIf(tree, [](auto const & e) { + return e.m_id == MakeId(9223372036938640141ull, 9223372036938640141ull); + }); + TEST(node, ()); + TEST_EQUAL(node->GetData().m_depth, tree_node::GetDepth(node), ()); + TEST_EQUAL(node->GetParent()->GetData().m_id, + MakeId(9223372036879747192ull, 9223372036879747192ull), ()); + TEST_EQUAL(node->GetChildren().size(), 0, ()); +} +} // namespace diff --git a/generator/hierarchy.cpp b/generator/hierarchy.cpp index ca07e5e20a..ce4022a1cd 100644 --- a/generator/hierarchy.cpp +++ b/generator/hierarchy.cpp @@ -154,12 +154,12 @@ HierarchyBuilder::HierarchyBuilder(std::string const & dataFilename) { } -void HierarchyBuilder::SetGetMainTypeFunction(GetMainType const & getMainType) +void HierarchyBuilder::SetGetMainTypeFunction(GetMainTypeFn const & getMainType) { m_getMainType = getMainType; } -void HierarchyBuilder::SetGetNameFunction(GetName const & getName) { m_getName = getName; } +void HierarchyBuilder::SetGetNameFunction(GetNameFn const & getName) { m_getName = getName; } std::vector HierarchyBuilder::ReadFeatures( std::string const & dataFilename) @@ -208,12 +208,12 @@ HierarchyLinesBuilder::HierarchyLinesBuilder(HierarchyBuilder::Node::PtrList && { } -void HierarchyLinesBuilder::SetGetMainTypeFunction(GetMainType const & getMainType) +void HierarchyLinesBuilder::SetGetMainTypeFunction(GetMainTypeFn const & getMainType) { m_getMainType = getMainType; } -void HierarchyLinesBuilder::SetGetNameFunction(GetName const & getName) { m_getName = getName; } +void HierarchyLinesBuilder::SetGetNameFunction(GetNameFn const & getName) { m_getName = getName; } void HierarchyLinesBuilder::SetCountryName(std::string const & name) { m_countryName = name; } diff --git a/generator/hierarchy.hpp b/generator/hierarchy.hpp index 7b96cbafd3..784d6b85e4 100644 --- a/generator/hierarchy.hpp +++ b/generator/hierarchy.hpp @@ -35,9 +35,9 @@ namespace generator { namespace hierarchy { -using GetMainType = std::function; -using GetName = std::function; -using PrintFunction = std::function; +using GetMainTypeFn = std::function; +using GetNameFn = std::function; +using PrintFn = std::function; // These are dummy functions. uint32_t GetTypeDefault(FeatureParams::Types const &); @@ -104,8 +104,8 @@ public: explicit HierarchyBuilder(std::string const & dataFilename); - void SetGetMainTypeFunction(GetMainType const & getMainType); - void SetGetNameFunction(GetName const & getName); + void SetGetMainTypeFunction(GetMainTypeFn const & getMainType); + void SetGetNameFunction(GetNameFn const & getName); Node::PtrList Build(); @@ -113,8 +113,8 @@ protected: std::vector ReadFeatures(std::string const & dataFilename); std::string m_dataFullFilename; - GetMainType m_getMainType = GetTypeDefault; - GetName m_getName = GetNameDefault; + GetMainTypeFn m_getMainType = GetTypeDefault; + GetNameFn m_getName = GetNameDefault; }; class HierarchyLineEnricher @@ -134,8 +134,8 @@ class HierarchyLinesBuilder public: HierarchyLinesBuilder(HierarchyBuilder::Node::PtrList && nodes); - void SetGetMainTypeFunction(GetMainType const & getMainType); - void SetGetNameFunction(GetName const & getName); + void SetGetMainTypeFunction(GetMainTypeFn const & getMainType); + void SetGetNameFunction(GetNameFn const & getName); void SetCountryName(std::string const & name); void SetHierarchyLineEnricher(std::shared_ptr const & enricher); @@ -146,8 +146,8 @@ private: HierarchyEntry Transform(HierarchyBuilder::Node::Ptr const & node); HierarchyBuilder::Node::PtrList m_nodes; - GetMainType m_getMainType = GetTypeDefault; - GetName m_getName = GetNameDefault; + GetMainTypeFn m_getMainType = GetTypeDefault; + GetNameFn m_getName = GetNameDefault; std::string m_countryName; std::shared_ptr m_enricher; }; diff --git a/generator/hierarchy_entry.cpp b/generator/hierarchy_entry.cpp index 62599f1914..637ae96a7c 100644 --- a/generator/hierarchy_entry.cpp +++ b/generator/hierarchy_entry.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "3party/jansson/myjansson.hpp" @@ -57,7 +58,7 @@ std::string DebugPrint(HierarchyEntry const & entry) return DumpToString(obj); } -namespace popularity +namespace hierarchy { uint32_t GetMainType(FeatureParams::Types const & types) { @@ -127,5 +128,32 @@ HierarchyEntry HierarchyEntryFromCsvRow(coding::CSVReader::Row const & row) entry.m_countryName = country; return entry; } -} // namespace popularity + +tree_node::types::PtrList LoadHierachy(std::string const & filename) +{ + std::unordered_map> nodes; + for (auto const & row : coding::CSVRunner( + coding::CSVReader(filename, false /* header */, kCsvDelimiter))) + { + auto entry = HierarchyEntryFromCsvRow(row); + auto const id = entry.m_id; + nodes.emplace(id, tree_node::MakeTreeNode(std::move(entry))); + } + for (auto const & pair : nodes) + { + auto const & node = pair.second; + auto const parentIdOpt = node->GetData().m_parentId; + if (parentIdOpt) + { + auto const it = nodes.find(*parentIdOpt); + CHECK(it != std::cend(nodes), (*it)); + tree_node::Link(node, it->second); + } + } + std::vector> trees; + base::Transform(nodes, std::back_inserter(trees), base::RetrieveSecond()); + base::EraseIf(trees, [](auto const & node) { return node->HasParent(); }); + return trees; +} +} // namespace hierarchy } // namespace generator diff --git a/generator/hierarchy_entry.hpp b/generator/hierarchy_entry.hpp index bb09b1969b..315d2acba5 100644 --- a/generator/hierarchy_entry.hpp +++ b/generator/hierarchy_entry.hpp @@ -3,6 +3,7 @@ #include "generator/composite_id.hpp" #include "indexer/classificator.hpp" +#include "indexer/complex/tree_node.hpp" #include "indexer/feature_data.hpp" #include "geometry/point2d.hpp" @@ -31,7 +32,7 @@ bool operator==(HierarchyEntry const & lhs, HierarchyEntry const & rhs); std::string DebugPrint(HierarchyEntry const & entry); -namespace popularity +namespace hierarchy { static char const kCsvDelimiter = ';'; @@ -42,5 +43,7 @@ coding::CSVReader::Row HierarchyEntryToCsvRow(HierarchyEntry const & entry); HierarchyEntry HierarchyEntryFromCsvRow(coding::CSVReader::Row const & row); std::string HierarchyEntryToCsvString(HierarchyEntry const & entry, char delim = kCsvDelimiter); -} // namespace popularity + +tree_node::types::PtrList LoadHierachy(std::string const & filename); +} // namespace hierarchy } // namespace generator diff --git a/indexer/complex/tree_node.hpp b/indexer/complex/tree_node.hpp index c04bd6731c..3bdfc9d306 100644 --- a/indexer/complex/tree_node.hpp +++ b/indexer/complex/tree_node.hpp @@ -5,6 +5,7 @@ #include #include +#include "base/control_flow.hpp" #include "base/stl_helpers.hpp" namespace tree_node @@ -58,6 +59,12 @@ private: WeakPtr m_parent; }; +template +decltype(auto) MakeTreeNode(Data && data) +{ + return std::make_shared>(std::forward(data)); +} + template void Link(types::Ptr const & node, types::Ptr const & parent) { @@ -76,4 +83,42 @@ size_t GetDepth(types::Ptr node) } return depth; } + +template +void PreOrderVisit(types::Ptr const & node, Fn && fn) +{ + base::ControlFlowWrapper wrapper(std::forward(fn)); + std::function const &)> preOrderVisitDetail; + preOrderVisitDetail = [&](auto const & node) { + if (wrapper(node) == base::ControlFlow::Break) + return; + + for (auto const & ch : node->GetChildren()) + preOrderVisitDetail(ch); + }; + preOrderVisitDetail(node); +} + +template +decltype(auto) FindIf(types::Ptr const & node, Fn && fn) +{ + types::Ptr res = nullptr; + PreOrderVisit(node, [&](auto const & node) { + if (fn(node->GetData())) + { + res = node; + return base::ControlFlow::Break; + } + return base::ControlFlow::Continue; + }); + return res; +} + +template +size_t Size(types::Ptr const & node) +{ + size_t size = 0; + PreOrderVisit(node, [&](auto const &) { ++size; }); + return size; +} } // namespace tree_node