From 2368208d37c1db0df40ef5128961ebace5d3b9be Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 13 Dec 2019 00:47:00 +0300 Subject: [PATCH] [generator] Added processing complex features. --- generator/feature_processing_layers.cpp | 83 +++++++++++++++++++ generator/feature_processing_layers.hpp | 27 +++++- .../final_processor_intermediate_mwm.cpp | 13 ++- .../final_processor_intermediate_mwm.hpp | 1 + generator/generate_info.hpp | 2 + generator/generator_tool/generator_tool.cpp | 2 + generator/processor_country.cpp | 9 +- generator/processor_country.hpp | 4 +- generator/raw_generator.cpp | 8 +- generator/raw_generator.hpp | 1 + indexer/feature_visibility.cpp | 5 +- 11 files changed, 143 insertions(+), 12 deletions(-) diff --git a/generator/feature_processing_layers.cpp b/generator/feature_processing_layers.cpp index 0cd01d0ffa..a45634f40f 100644 --- a/generator/feature_processing_layers.cpp +++ b/generator/feature_processing_layers.cpp @@ -71,8 +71,16 @@ std::shared_ptr LayerBase::Add(std::shared_ptr next) return next; } +RepresentationLayer::RepresentationLayer(std::shared_ptr const & complexFeaturesMixer) + : m_complexFeaturesMixer(complexFeaturesMixer) +{ +} + void RepresentationLayer::Handle(FeatureBuilder & fb) { + if (m_complexFeaturesMixer) + m_complexFeaturesMixer->Process([&](FeatureBuilder & fb){ LayerBase::Handle(fb); }, fb); + auto const sourceType = fb.GetMostGenericOsmId().GetType(); auto const geomType = fb.GetGeomType(); // There is a copy of params here, if there is a reference here, then the params can be @@ -236,4 +244,79 @@ void PreserializeLayer::Handle(FeatureBuilder & fb) if (fb.PreSerialize()) LayerBase::Handle(fb); } + +ComplexFeaturesMixer::ComplexFeaturesMixer(std::unordered_set const & hierarchyNodesSet) + : m_hierarchyNodesSet(hierarchyNodesSet) + , m_complexEntryType(classif().GetTypeByPath({"complex_entry"})) +{ +} + +std::shared_ptr ComplexFeaturesMixer::Clone() +{ + return std::make_shared(m_hierarchyNodesSet); +} + +void ComplexFeaturesMixer::Process(std::function next, + feature::FeatureBuilder const & fb) +{ + if (!next) + return; + + // For all objects in the hierarchy, there must be one areal object and one linear. + // Exceptions are point features and parts of buildings. + if (fb.IsPoint() || !fb.IsGeometryClosed()) + return; + + auto const id = MakeCompositeId(fb); + auto const it = m_hierarchyNodesSet.find(id); + if (it == std::end(m_hierarchyNodesSet)) + return; + + static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance(); + if (buildingPartChecker(fb.GetTypes())) + return; + + auto const canBeArea = RepresentationLayer::CanBeArea(fb.GetParams()); + auto const canBeLine = RepresentationLayer::CanBeLine(fb.GetParams()); + if (!canBeArea) + { + LOG(LINFO, ("Add a areal complex feature for", fb.GetMostGenericOsmId())); + auto complexFb = MakeComplexAreaFrom(fb); + next(complexFb); + } + + if (!canBeLine) + { + LOG(LINFO, ("Add a linear complex feature for", fb.GetMostGenericOsmId())); + auto complexFb = MakeComplexLineFrom(fb); + next(complexFb); + } +} + +feature::FeatureBuilder ComplexFeaturesMixer::MakeComplexLineFrom(feature::FeatureBuilder const & fb) +{ + CHECK(fb.IsArea() || fb.IsLine(), ()); + CHECK(fb.IsGeometryClosed(), ()); + + auto lineFb = MakeLine(fb); + auto & params = lineFb.GetParams(); + params.ClearName(); + params.GetMetadata() = {}; + params.SetType(m_complexEntryType); + return lineFb; +} + +feature::FeatureBuilder ComplexFeaturesMixer::MakeComplexAreaFrom(feature::FeatureBuilder const & fb) +{ + CHECK(fb.IsArea() || fb.IsLine(), ()); + CHECK(fb.IsGeometryClosed(), ()); + + auto areaFb = fb; + areaFb.SetArea(); + auto & params = areaFb.GetParams(); + params.ClearName(); + params.GetMetadata() = {}; + params.SetType(m_complexEntryType); + return areaFb; +} } // namespace generator diff --git a/generator/feature_processing_layers.hpp b/generator/feature_processing_layers.hpp index fdca57bdeb..27e50ab916 100644 --- a/generator/feature_processing_layers.hpp +++ b/generator/feature_processing_layers.hpp @@ -24,7 +24,7 @@ struct GenerateInfo; namespace generator { -class PlaceProcessor; +class ComplexFeaturesMixer; // This is the base layer class. Inheriting from it allows you to create a chain of layers. class LayerBase : public std::enable_shared_from_this { @@ -52,15 +52,20 @@ private: // with type "leisure=playground" and line object with type "barrier=fence". class RepresentationLayer : public LayerBase { +public: + explicit RepresentationLayer(std::shared_ptr const & complexFeaturesMixer = nullptr); + // LayerBase overrides: void Handle(feature::FeatureBuilder & fb) override; -private: static bool CanBeArea(FeatureParams const & params); static bool CanBePoint(FeatureParams const & params); static bool CanBeLine(FeatureParams const & params); +private: void HandleArea(feature::FeatureBuilder & fb, FeatureBuilderParams const & params); + + std::shared_ptr m_complexFeaturesMixer; }; // Responsibility of class PrepareFeatureLayer is the removal of unused types and names, @@ -156,4 +161,22 @@ private: std::shared_ptr m_affiliation; std::shared_ptr m_queue; }; + +class ComplexFeaturesMixer +{ +public: + explicit ComplexFeaturesMixer(std::unordered_set const & hierarchyNodesSet); + + void Process(std::function next, + feature::FeatureBuilder const & fb); + + std::shared_ptr Clone(); + +private: + feature::FeatureBuilder MakeComplexLineFrom(feature::FeatureBuilder const & fb); + feature::FeatureBuilder MakeComplexAreaFrom(feature::FeatureBuilder const & fb); + + std::unordered_set const & m_hierarchyNodesSet; + uint32_t const m_complexEntryType; +}; } // namespace generator diff --git a/generator/final_processor_intermediate_mwm.cpp b/generator/final_processor_intermediate_mwm.cpp index 013f24f033..687f950232 100644 --- a/generator/final_processor_intermediate_mwm.cpp +++ b/generator/final_processor_intermediate_mwm.cpp @@ -2,6 +2,7 @@ #include "generator/affiliation.hpp" #include "generator/booking_dataset.hpp" +#include "generator/complex_loader.hpp" #include "generator/feature_merger.hpp" #include "generator/mini_roundabout_transformer.hpp" #include "generator/node_mixer.hpp" @@ -119,10 +120,15 @@ void Sort(std::vector & fbs) }); } -bool FilenameIsCountry(std::string filename, AffiliationInterface const & affiliation) +std::string GetCountryNameFormTmpMwmPath(std::string filename) { strings::ReplaceLast(filename, DATA_FILE_EXTENSION_TMP, ""); - return affiliation.HasRegionByName(filename); + return filename; +} + +bool FilenameIsCountry(std::string const & filename, AffiliationInterface const & affiliation) +{ + return affiliation.HasRegionByName(GetCountryNameFormTmpMwmPath(filename)); } class PlaceHelper @@ -638,8 +644,7 @@ void ComplexFinalProcessor::Process() std::vector>> futures; ForEachCountry(m_mwmTmpPath, [&](auto const & filename) { auto future = pool.Submit([&, filename]() { - auto countryName = filename; - strings::ReplaceLast(countryName, DATA_FILE_EXTENSION_TMP, ""); + auto countryName = GetCountryNameFormTmpMwmPath(filename); // https://wiki.openstreetmap.org/wiki/Simple_3D_buildings // An object with tag 'building:part' is a part of a relation with outline 'building' or // is contained in an object with tag 'building'. We will split data and work with diff --git a/generator/final_processor_intermediate_mwm.hpp b/generator/final_processor_intermediate_mwm.hpp index 30d6538ab3..919815cead 100644 --- a/generator/final_processor_intermediate_mwm.hpp +++ b/generator/final_processor_intermediate_mwm.hpp @@ -88,6 +88,7 @@ private: std::string m_routingCityBoundariesCollectorFilename; std::string m_routingCityBoundariesDumpPath; + std::string m_hierarchySrcFilename; bool m_haveBordersForWholeWorld; size_t m_threadsCount; diff --git a/generator/generate_info.hpp b/generator/generate_info.hpp index 6d5d6ec843..405b75346f 100644 --- a/generator/generate_info.hpp +++ b/generator/generate_info.hpp @@ -57,6 +57,8 @@ struct GenerateInfo std::string m_citiesBoundariesFilename; + std::string m_complexHierarchyFilename; + uint32_t m_versionDate = 0; std::vector m_bucketNames; diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 26276fc210..1837a49f8d 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -151,6 +151,7 @@ DEFINE_string(booking_data, "", "Path to booking data in tsv format."); DEFINE_string(opentable_data, "", "Path to opentable data in tsv format."); DEFINE_string(promo_catalog_cities, "", "Path to list geo object ids of cities which contain promo catalog in json format."); +DEFINE_string(complex_hierarchy_data, "", "Path to complex hierarchy in csv format."); DEFINE_string(ugc_data, "", "Input UGC source database file name."); @@ -257,6 +258,7 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) genInfo.m_emitCoasts = FLAGS_emit_coasts; genInfo.m_fileName = FLAGS_output; genInfo.m_idToWikidataFilename = FLAGS_idToWikidata; + genInfo.m_complexHierarchyFilename = FLAGS_complex_hierarchy_data; // Use merged style. GetStyleReader().SetCurrentStyle(MapStyleMerged); diff --git a/generator/processor_country.cpp b/generator/processor_country.cpp index d65506a70c..a122cbc926 100644 --- a/generator/processor_country.cpp +++ b/generator/processor_country.cpp @@ -10,12 +10,14 @@ namespace generator { ProcessorCountry::ProcessorCountry(std::shared_ptr const & queue, - std::string const & bordersPath, bool haveBordersForWholeWorld) + std::string const & bordersPath, bool haveBordersForWholeWorld, + std::shared_ptr const & complexFeaturesMixer) : m_bordersPath(bordersPath) , m_queue(queue) + , m_complexFeaturesMixer(complexFeaturesMixer) , m_haveBordersForWholeWorld(haveBordersForWholeWorld) { - m_processingChain = std::make_shared(); + m_processingChain = std::make_shared(m_complexFeaturesMixer); m_processingChain->Add(std::make_shared()); m_processingChain->Add(std::make_shared()); auto affiliation = std::make_shared( @@ -27,7 +29,8 @@ ProcessorCountry::ProcessorCountry(std::shared_ptr const std::shared_ptr ProcessorCountry::Clone() const { - return std::make_shared(m_queue, m_bordersPath, m_haveBordersForWholeWorld); + return std::make_shared(m_queue, m_bordersPath, m_haveBordersForWholeWorld, + m_complexFeaturesMixer->Clone()); } void ProcessorCountry::Process(feature::FeatureBuilder & feature) diff --git a/generator/processor_country.hpp b/generator/processor_country.hpp index 962a5c7c5b..0dbb936d0a 100644 --- a/generator/processor_country.hpp +++ b/generator/processor_country.hpp @@ -20,7 +20,8 @@ class ProcessorCountry : public FeatureProcessorInterface { public: explicit ProcessorCountry(std::shared_ptr const & queue, - std::string const & bordersPath, bool haveBordersForWholeWorld); + std::string const & bordersPath, bool haveBordersForWholeWorld, + std::shared_ptr const & complexFeaturesMixer); // FeatureProcessorInterface overrides: std::shared_ptr Clone() const override; @@ -33,6 +34,7 @@ private: std::shared_ptr> m_affiliationsLayer; std::shared_ptr m_queue; std::shared_ptr m_processingChain; + std::shared_ptr m_complexFeaturesMixer; bool m_haveBordersForWholeWorld; }; } // namespace generator diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index 59705a61e6..05a3918d7b 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -1,5 +1,6 @@ #include "generator/raw_generator.hpp" +#include "generator/complex_loader.hpp" #include "generator/osm_source.hpp" #include "generator/processor_factory.hpp" #include "generator/raw_generator_writer.hpp" @@ -31,8 +32,13 @@ std::shared_ptr RawGenerator::GetQueue() { return m_queue void RawGenerator::GenerateCountries(bool addAds) { + if (!m_genInfo.m_complexHierarchyFilename.empty()) + m_hierarchyNodesSet = GetOrCreateComplexLoader(m_genInfo.m_complexHierarchyFilename).GetIdsSet(); + + auto const complexFeaturesMixer = std::make_shared(m_hierarchyNodesSet); + auto processor = CreateProcessor(ProcessorType::Country, m_queue, m_genInfo.m_targetDir, - m_genInfo.m_haveBordersForWholeWorld); + m_genInfo.m_haveBordersForWholeWorld, complexFeaturesMixer); m_translators->Append( CreateTranslator(TranslatorType::Country, processor, m_cache, m_genInfo, addAds)); m_finalProcessors.emplace(CreateCountryFinalProcessor(addAds)); diff --git a/generator/raw_generator.hpp b/generator/raw_generator.hpp index e4d01a4c1c..f81e1bd2bf 100644 --- a/generator/raw_generator.hpp +++ b/generator/raw_generator.hpp @@ -54,5 +54,6 @@ private: std::priority_queue, FinalProcessorPtrCmp> m_finalProcessors; std::vector m_names; + std::unordered_set m_hierarchyNodesSet; }; } // namespace generator diff --git a/indexer/feature_visibility.cpp b/indexer/feature_visibility.cpp index 2365767295..a158f0c96d 100644 --- a/indexer/feature_visibility.cpp +++ b/indexer/feature_visibility.cpp @@ -229,6 +229,7 @@ namespace return false; static uint32_t const internet = classif().GetTypeByPath({"internet_access"}); + static uint32_t const complex_entry = classif().GetTypeByPath({"complex_entry"}); if ((g == GeomType::Line || g == GeomType::Undefined) && HasRoutingExceptionType(type)) return true; @@ -237,6 +238,9 @@ namespace if (g != GeomType::Line && type == internet) return true; + if (type == complex_entry) + return true; + return false; } @@ -284,7 +288,6 @@ namespace // Reserved for custom event processing, e.g. fc2018. // if (event == type) // return true; - return false; } } // namespace