From d3c46ddb9409db95fd156e1b61629c58cfc921b6 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 17 Apr 2020 03:42:43 +0300 Subject: [PATCH] [generator] Files generator/final_processor_intermediate_mwm.* were splitted into several files generator/final_processor_[processor type].*. --- generator/CMakeLists.txt | 14 +- .../complex_generator/complex_generator.cpp | 2 +- generator/final_processor_coastline.cpp | 53 ++ generator/final_processor_coastline.hpp | 27 + generator/final_processor_complex.cpp | 171 ++++ generator/final_processor_complex.hpp | 57 ++ generator/final_processor_country.cpp | 276 +++++++ generator/final_processor_country.hpp | 61 ++ generator/final_processor_interface.cpp | 28 + generator/final_processor_interface.hpp | 36 + .../final_processor_intermediate_mwm.cpp | 773 ------------------ .../final_processor_intermediate_mwm.hpp | 184 ----- generator/final_processor_utils.cpp | 141 ++++ generator/final_processor_utils.hpp | 123 +++ generator/final_processor_world.cpp | 61 ++ generator/final_processor_world.hpp | 36 + .../features_tests.cpp | 1 + generator/raw_generator.cpp | 4 + generator/raw_generator.hpp | 3 +- 19 files changed, 1090 insertions(+), 961 deletions(-) create mode 100644 generator/final_processor_coastline.cpp create mode 100644 generator/final_processor_coastline.hpp create mode 100644 generator/final_processor_complex.cpp create mode 100644 generator/final_processor_complex.hpp create mode 100644 generator/final_processor_country.cpp create mode 100644 generator/final_processor_country.hpp create mode 100644 generator/final_processor_interface.cpp create mode 100644 generator/final_processor_interface.hpp delete mode 100644 generator/final_processor_intermediate_mwm.cpp delete mode 100644 generator/final_processor_intermediate_mwm.hpp create mode 100644 generator/final_processor_utils.cpp create mode 100644 generator/final_processor_utils.hpp create mode 100644 generator/final_processor_world.cpp create mode 100644 generator/final_processor_world.hpp diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index 1837bced70..15a7198030 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -98,8 +98,18 @@ set( filter_roads.hpp filter_world.cpp filter_world.hpp - final_processor_intermediate_mwm.cpp - final_processor_intermediate_mwm.hpp + final_processor_coastline.cpp + final_processor_coastline.hpp + final_processor_complex.cpp + final_processor_complex.hpp + final_processor_country.cpp + final_processor_country.hpp + final_processor_interface.cpp + final_processor_interface.hpp + final_processor_utils.cpp + final_processor_utils.hpp + final_processor_world.cpp + final_processor_world.hpp gen_mwm_info.cpp gen_mwm_info.hpp generate_info.hpp diff --git a/generator/complex_generator/complex_generator.cpp b/generator/complex_generator/complex_generator.cpp index 47e06d9730..e0e5cbe25d 100644 --- a/generator/complex_generator/complex_generator.cpp +++ b/generator/complex_generator/complex_generator.cpp @@ -17,7 +17,7 @@ #include "generator/filter_complex.hpp" #include "generator/filter_interface.hpp" -#include "generator/final_processor_intermediate_mwm.hpp" +#include "generator/final_processor_complex.hpp" #include "generator/generate_info.hpp" #include "generator/hierarchy.hpp" #include "generator/hierarchy_entry.hpp" diff --git a/generator/final_processor_coastline.cpp b/generator/final_processor_coastline.cpp new file mode 100644 index 0000000000..5b971bd2b3 --- /dev/null +++ b/generator/final_processor_coastline.cpp @@ -0,0 +1,53 @@ +#include "generator/final_processor_coastline.hpp" + +#include "generator/feature_builder.hpp" +#include "generator/feature_generator.hpp" +#include "generator/final_processor_utils.hpp" + +#include + +using namespace feature; + +namespace generator +{ +CoastlineFinalProcessor::CoastlineFinalProcessor(std::string const & filename) + : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::WorldCoasts) + , m_filename(filename) +{ +} + +void CoastlineFinalProcessor::SetCoastlinesFilenames(std::string const & geomFilename, + std::string const & rawGeomFilename) +{ + m_coastlineGeomFilename = geomFilename; + m_coastlineRawGeomFilename = rawGeomFilename; +} + +void CoastlineFinalProcessor::Process() +{ + auto fbs = ReadAllDatRawFormat(m_filename); + Sort(fbs); + for (auto && fb : fbs) + m_generator.Process(std::move(fb)); + + FeaturesAndRawGeometryCollector collector(m_coastlineGeomFilename, m_coastlineRawGeomFilename); + // Check and stop if some coasts were not merged. + CHECK(m_generator.Finish(), ()); + LOG(LINFO, ("Generating coastline polygons.")); + size_t totalFeatures = 0; + size_t totalPoints = 0; + size_t totalPolygons = 0; + std::vector outputFbs; + m_generator.GetFeatures(outputFbs); + for (auto & fb : outputFbs) + { + collector.Collect(fb); + ++totalFeatures; + totalPoints += fb.GetPointsCount(); + totalPolygons += fb.GetPolygonsCount(); + } + + LOG(LINFO, ("Total features:", totalFeatures, "total polygons:", totalPolygons, + "total points:", totalPoints)); +} +} // namespace generator diff --git a/generator/final_processor_coastline.hpp b/generator/final_processor_coastline.hpp new file mode 100644 index 0000000000..d4a198f155 --- /dev/null +++ b/generator/final_processor_coastline.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "generator/coastlines_generator.hpp" +#include "generator/final_processor_interface.hpp" + +#include + +namespace generator +{ +class CoastlineFinalProcessor : public FinalProcessorIntermediateMwmInterface +{ +public: + explicit CoastlineFinalProcessor(std::string const & filename); + + void SetCoastlinesFilenames(std::string const & geomFilename, + std::string const & rawGeomFilename); + + // FinalProcessorIntermediateMwmInterface overrides: + void Process() override; + +private: + std::string m_filename; + std::string m_coastlineGeomFilename; + std::string m_coastlineRawGeomFilename; + CoastlineFeaturesGenerator m_generator; +}; +} // namespace generator diff --git a/generator/final_processor_complex.cpp b/generator/final_processor_complex.cpp new file mode 100644 index 0000000000..8855e32add --- /dev/null +++ b/generator/final_processor_complex.cpp @@ -0,0 +1,171 @@ +#include "generator/final_processor_complex.hpp" + +#include "generator/final_processor_utils.hpp" +#include "generator/utils.hpp" + +#include "base/file_name_utils.hpp" +#include "base/thread_pool_computational.hpp" + +using namespace feature; + +namespace generator +{ +ComplexFinalProcessor::ComplexFinalProcessor(std::string const & mwmTmpPath, + std::string const & outFilename, size_t threadsCount) + : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::Complex) + , m_mwmTmpPath(mwmTmpPath) + , m_outFilename(outFilename) + , m_threadsCount(threadsCount) +{ +} + +void ComplexFinalProcessor::SetGetMainTypeFunction(hierarchy::GetMainTypeFn const & getMainType) +{ + m_getMainType = getMainType; +} + +void ComplexFinalProcessor::SetFilter(std::shared_ptr const & filter) +{ + m_filter = filter; +} + +void ComplexFinalProcessor::SetGetNameFunction(hierarchy::GetNameFn const & getName) +{ + m_getName = getName; +} + +void ComplexFinalProcessor::SetPrintFunction(hierarchy::PrintFn const & printFunction) +{ + m_printFunction = printFunction; +} + +void ComplexFinalProcessor::UseCentersEnricher(std::string const & mwmPath, + std::string const & osm2ftPath) +{ + m_useCentersEnricher = true; + m_mwmPath = mwmPath; + m_osm2ftPath = osm2ftPath; +} + +std::unique_ptr ComplexFinalProcessor::CreateEnricher( + std::string const & countryName) const +{ + return std::make_unique( + base::JoinPath(m_osm2ftPath, countryName + DATA_FILE_EXTENSION OSM2FEATURE_FILE_EXTENSION), + base::JoinPath(m_mwmPath, countryName + DATA_FILE_EXTENSION)); +} + +void ComplexFinalProcessor::UseBuildingPartsInfo(std::string const & filename) +{ + m_buildingPartsFilename = filename; +} + +void ComplexFinalProcessor::Process() +{ + if (!m_buildingPartsFilename.empty()) + m_buildingToParts = std::make_unique(m_buildingPartsFilename); + + base::thread_pool::computational::ThreadPool pool(m_threadsCount); + std::vector>> futures; + ForEachCountry(m_mwmTmpPath, [&](auto const & filename) { + auto future = pool.Submit([&, filename]() { + auto countryName = GetCountryNameFromTmpMwmPath(filename); + // https://wiki.openstreetmap.org/wiki/Simple_3D_buildings + // An object with tag 'building:part' is a part of a relation with outline 'building' or + // is contained in an object with tag 'building'. We will split data and work with + // these cases separately. First of all let's remove objects with tag building:part is + // contained in relations. We will add them back after data processing. + std::unordered_map relationBuildingParts; + + auto fbs = ReadAllDatRawFormat( + base::JoinPath(m_mwmTmpPath, filename)); + if (m_buildingToParts) + relationBuildingParts = RemoveRelationBuildingParts(fbs); + + // This case is second. We build a hierarchy using the geometry of objects and their nesting. + auto trees = hierarchy::BuildHierarchy(std::move(fbs), m_getMainType, m_filter); + + // We remove tree roots with tag 'building:part'. + base::EraseIf(trees, [](auto const & node) { + static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance(); + return buildingPartChecker(node->GetData().GetTypes()); + }); + + // We want to transform + // building + // |_building-part + // |_building-part + // to + // building + // |_building-part + // |_building-part + hierarchy::FlattenBuildingParts(trees); + // In the end we add objects, which were saved by the collector. + if (m_buildingToParts) + { + hierarchy::AddChildrenTo(trees, [&](auto const & compositeId) { + auto const & ids = m_buildingToParts->GetBuildingPartsByOutlineId(compositeId); + std::vector places; + places.reserve(ids.size()); + for (auto const & id : ids) + { + if (relationBuildingParts.count(id) == 0) + continue; + + places.emplace_back(hierarchy::HierarchyPlace(relationBuildingParts[id])); + } + return places; + }); + } + + // We create and save hierarchy lines. + hierarchy::HierarchyLinesBuilder hierarchyBuilder(std::move(trees)); + if (m_useCentersEnricher) + hierarchyBuilder.SetHierarchyEntryEnricher(CreateEnricher(countryName)); + + hierarchyBuilder.SetCountry(countryName); + hierarchyBuilder.SetGetMainTypeFunction(m_getMainType); + hierarchyBuilder.SetGetNameFunction(m_getName); + return hierarchyBuilder.GetHierarchyLines(); + }); + futures.emplace_back(std::move(future)); + }); + std::vector allLines; + for (auto & f : futures) + { + auto const lines = f.get(); + allLines.insert(std::cend(allLines), std::cbegin(lines), std::cend(lines)); + } + WriteLines(allLines); +} + +std::unordered_map +ComplexFinalProcessor::RemoveRelationBuildingParts(std::vector & fbs) +{ + CHECK(m_buildingToParts, ()); + + auto it = std::partition(std::begin(fbs), std::end(fbs), [&](auto const & fb) { + return !m_buildingToParts->HasBuildingPart(fb.GetMostGenericOsmId()); + }); + + std::unordered_map buildingParts; + buildingParts.reserve(static_cast(std::distance(it, std::end(fbs)))); + + std::transform(it, std::end(fbs), std::inserter(buildingParts, std::begin(buildingParts)), + [](auto && fb) { + auto const id = fb.GetMostGenericOsmId(); + return std::make_pair(id, std::move(fb)); + }); + + fbs.resize(static_cast(std::distance(std::begin(fbs), it))); + return buildingParts; +} + +void ComplexFinalProcessor::WriteLines(std::vector const & lines) +{ + ; + auto stream = OfstreamWithExceptions(m_outFilename); + for (auto const & line : lines) + stream << m_printFunction(line) << '\n'; +} +} // namespace generator diff --git a/generator/final_processor_complex.hpp b/generator/final_processor_complex.hpp new file mode 100644 index 0000000000..eea730639a --- /dev/null +++ b/generator/final_processor_complex.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "generator/collector_building_parts.hpp" +#include "generator/feature_builder.hpp" +#include "generator/filter_interface.hpp" +#include "generator/final_processor_interface.hpp" +#include "generator/hierarchy.hpp" +#include "generator/hierarchy_entry.hpp" + +#include +#include +#include +#include + +namespace generator +{ +// Class ComplexFinalProcessor generates hierarchies for each previously filtered mwm.tmp file. +// Warning: If the border separates the complex, then a situation is possible in which two logically +// identical complexes are generated, but with different representations. +class ComplexFinalProcessor : public FinalProcessorIntermediateMwmInterface +{ +public: + ComplexFinalProcessor(std::string const & mwmTmpPath, std::string const & outFilename, + size_t threadsCount); + + void SetGetMainTypeFunction(hierarchy::GetMainTypeFn const & getMainType); + void SetFilter(std::shared_ptr const & filter); + void SetGetNameFunction(hierarchy::GetNameFn const & getName); + void SetPrintFunction(hierarchy::PrintFn const & printFunction); + + void UseCentersEnricher(std::string const & mwmPath, std::string const & osm2ftPath); + void UseBuildingPartsInfo(std::string const & filename); + + // FinalProcessorIntermediateMwmInterface overrides: + void Process() override; + +private: + std::unique_ptr CreateEnricher( + std::string const & countryName) const; + void WriteLines(std::vector const & lines); + std::unordered_map RemoveRelationBuildingParts( + std::vector & fbs); + + hierarchy::GetMainTypeFn m_getMainType; + hierarchy::PrintFn m_printFunction; + hierarchy::GetNameFn m_getName; + std::shared_ptr m_filter; + std::unique_ptr m_buildingToParts; + bool m_useCentersEnricher = false; + std::string m_mwmTmpPath; + std::string m_outFilename; + std::string m_mwmPath; + std::string m_osm2ftPath; + std::string m_buildingPartsFilename; + size_t m_threadsCount; +}; +} // namespace generator diff --git a/generator/final_processor_country.cpp b/generator/final_processor_country.cpp new file mode 100644 index 0000000000..86bcf75784 --- /dev/null +++ b/generator/final_processor_country.cpp @@ -0,0 +1,276 @@ +#include "generator/final_processor_country.hpp" + +#include "generator/booking_dataset.hpp" +#include "generator/feature_builder.hpp" +#include "generator/final_processor_utils.hpp" +#include "generator/isolines_generator.hpp" +#include "generator/mini_roundabout_transformer.hpp" +#include "generator/node_mixer.hpp" +#include "generator/osm2type.hpp" +#include "generator/routing_city_boundaries_processor.hpp" + +#include "routing/speed_camera_prohibition.hpp" + +#include "indexer/classificator.hpp" +#include "indexer/feature_algo.hpp" + +#include "base/file_name_utils.hpp" +#include "base/thread_pool_computational.hpp" + +#include +#include + +using namespace base::thread_pool::computational; +using namespace feature; + +namespace generator +{ +CountryFinalProcessor::CountryFinalProcessor(std::string const & borderPath, + std::string const & temporaryMwmPath, + bool haveBordersForWholeWorld, size_t threadsCount) + : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::CountriesOrWorld) + , m_borderPath(borderPath) + , m_temporaryMwmPath(temporaryMwmPath) + , m_haveBordersForWholeWorld(haveBordersForWholeWorld) + , m_threadsCount(threadsCount) +{ +} + +void CountryFinalProcessor::SetBooking(std::string const & filename) +{ + m_hotelsFilename = filename; +} + +void CountryFinalProcessor::SetCitiesAreas(std::string const & filename) +{ + m_citiesAreasTmpFilename = filename; +} + +void CountryFinalProcessor::SetPromoCatalog(std::string const & filename) +{ + m_citiesFilename = filename; +} + +void CountryFinalProcessor::DumpCitiesBoundaries(std::string const & filename) +{ + m_citiesBoundariesFilename = filename; +} + +void CountryFinalProcessor::DumpRoutingCitiesBoundaries(std::string const & collectorFilename, + std::string const & dumpPath) +{ + m_routingCityBoundariesCollectorFilename = collectorFilename; + m_routingCityBoundariesDumpPath = dumpPath; +} + +void CountryFinalProcessor::SetCoastlines(std::string const & coastlineGeomFilename, + std::string const & worldCoastsFilename) +{ + m_coastlineGeomFilename = coastlineGeomFilename; + m_worldCoastsFilename = worldCoastsFilename; +} + +void CountryFinalProcessor::SetFakeNodes(std::string const & filename) +{ + m_fakeNodesFilename = filename; +} + +void CountryFinalProcessor::SetMiniRoundabouts(std::string const & filename) +{ + m_miniRoundaboutsFilename = filename; +} + +void CountryFinalProcessor::SetIsolinesDir(std::string const & dir) { m_isolinesPath = dir; } + +void CountryFinalProcessor::Process() +{ + if (!m_hotelsFilename.empty()) + ProcessBooking(); + if (!m_routingCityBoundariesCollectorFilename.empty()) + ProcessRoutingCityBoundaries(); + if (!m_citiesAreasTmpFilename.empty() || !m_citiesFilename.empty()) + ProcessCities(); + if (!m_coastlineGeomFilename.empty()) + ProcessCoastline(); + if (!m_miniRoundaboutsFilename.empty()) + ProcessRoundabouts(); + if (!m_fakeNodesFilename.empty()) + AddFakeNodes(); + if (!m_isolinesPath.empty()) + AddIsolines(); + Finish(); +} + +void CountryFinalProcessor::ProcessBooking() +{ + BookingDataset dataset(m_hotelsFilename); + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + { + ThreadPool pool(m_threadsCount); + ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { + pool.SubmitWork([&, filename]() { + std::vector cities; + if (!FilenameIsCountry(filename, affiliation)) + return; + + auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); + auto fbs = ReadAllDatRawFormat(fullPath); + FeatureBuilderWriter writer(fullPath); + for (auto & fb : fbs) + { + auto const id = dataset.FindMatchingObjectId(fb); + if (id == BookingHotel::InvalidObjectId()) + { + writer.Write(fb); + } + else + { + dataset.PreprocessMatchedOsmObject(id, fb, [&](FeatureBuilder & newFeature) { + if (newFeature.PreSerialize()) + writer.Write(newFeature); + }); + } + } + }); + }); + } + std::vector fbs; + dataset.BuildOsmObjects([&](auto && fb) { fbs.emplace_back(std::move(fb)); }); + auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); + AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); +} + +void CountryFinalProcessor::ProcessRoundabouts() +{ + MiniRoundaboutTransformer helper(m_miniRoundaboutsFilename); + + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + { + ThreadPool pool(m_threadsCount); + ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { + pool.SubmitWork([&, filename]() { + if (!FilenameIsCountry(filename, affiliation)) + return; + + auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); + auto fbs = ReadAllDatRawFormat(fullPath); + FeatureBuilderWriter writer(fullPath); + + // Adds new way features generated from mini-roundabout nodes with those nodes ids. + // Transforms points on roads to connect them with these new roundabout junctions. + helper.ProcessRoundabouts(affiliation, fbs); + for (auto const & fb : fbs) + writer.Write(fb); + }); + }); + } +} + +void CountryFinalProcessor::AddIsolines() +{ + // For generated isolines must be built isolines_info section based on the same + // binary isolines file. + IsolineFeaturesGenerator isolineFeaturesGenerator(m_isolinesPath); + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + ThreadPool pool(m_threadsCount); + ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { + pool.SubmitWork([&, filename]() { + if (!FilenameIsCountry(filename, affiliation)) + return; + auto const countryName = GetCountryNameFromTmpMwmPath(filename); + + auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); + FeatureBuilderWriter writer(fullPath, + FileWriter::Op::OP_APPEND); + isolineFeaturesGenerator.GenerateIsolines( + countryName, [&writer](feature::FeatureBuilder && fb) { writer.Write(fb); }); + }); + }); +} + +void CountryFinalProcessor::ProcessRoutingCityBoundaries() +{ + CHECK( + !m_routingCityBoundariesCollectorFilename.empty() && !m_routingCityBoundariesDumpPath.empty(), + ()); + + RoutingCityBoundariesProcessor processor(m_routingCityBoundariesCollectorFilename, + m_routingCityBoundariesDumpPath); + processor.ProcessDataFromCollector(); +} + +void CountryFinalProcessor::ProcessCities() +{ + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + auto citiesHelper = + m_citiesAreasTmpFilename.empty() ? PlaceHelper() : PlaceHelper(m_citiesAreasTmpFilename); + ProcessorCities processorCities(m_temporaryMwmPath, affiliation, citiesHelper, m_threadsCount); + processorCities.SetPromoCatalog(m_citiesFilename); + processorCities.Process(); + if (m_citiesBoundariesFilename.empty()) + return; + + auto const citiesTable = citiesHelper.GetTable(); + LOG(LINFO, ("Dumping cities boundaries to", m_citiesBoundariesFilename)); + SerializeBoundariesTable(m_citiesBoundariesFilename, *citiesTable); +} + +void CountryFinalProcessor::ProcessCoastline() +{ + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + auto fbs = ReadAllDatRawFormat(m_coastlineGeomFilename); + auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); + AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); + FeatureBuilderWriter<> collector(m_worldCoastsFilename); + for (size_t i = 0; i < fbs.size(); ++i) + { + fbs[i].AddName("default", strings::JoinStrings(affiliations[i], ';')); + collector.Write(fbs[i]); + } +} + +void CountryFinalProcessor::AddFakeNodes() +{ + std::vector fbs; + MixFakeNodes(m_fakeNodesFilename, [&](auto & element) { + FeatureBuilder fb; + fb.SetCenter(mercator::FromLatLon(element.m_lat, element.m_lon)); + fb.SetOsmId(base::MakeOsmNode(element.m_id)); + ftype::GetNameAndType(&element, fb.GetParams()); + fbs.emplace_back(std::move(fb)); + }); + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); + AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); +} + +void CountryFinalProcessor::Finish() +{ + auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); + ThreadPool pool(m_threadsCount); + ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { + pool.SubmitWork([&, filename]() { + if (!FilenameIsCountry(filename, affiliation)) + return; + + auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); + auto fbs = ReadAllDatRawFormat(fullPath); + Sort(fbs); + + auto const speedCamerasProhibited = + routing::AreSpeedCamerasProhibited(platform::CountryFile(filename)); + static auto const speedCameraType = classif().GetTypeByPath({"highway", "speed_camera"}); + + FeatureBuilderWriter<> collector(fullPath); + for (auto & fb : fbs) + { + // Removing point features with speed cameras type from geometry index for some countries. + if (speedCamerasProhibited && fb.IsPoint() && fb.HasType(speedCameraType)) + continue; + + collector.Write(fb); + } + }); + }); +} +} // namespace generator diff --git a/generator/final_processor_country.hpp b/generator/final_processor_country.hpp new file mode 100644 index 0000000000..8574dbee33 --- /dev/null +++ b/generator/final_processor_country.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include "generator/final_processor_interface.hpp" + +#include +#include + +namespace generator +{ +class CountryFinalProcessor : public FinalProcessorIntermediateMwmInterface +{ +public: + CountryFinalProcessor(std::string const & borderPath, std::string const & temporaryMwmPath, + bool haveBordersForWholeWorld, size_t threadsCount); + + void SetBooking(std::string const & filename); + void SetCitiesAreas(std::string const & filename); + void SetPromoCatalog(std::string const & filename); + void SetCoastlines(std::string const & coastlineGeomFilename, + std::string const & worldCoastsFilename); + void SetFakeNodes(std::string const & filename); + void SetMiniRoundabouts(std::string const & filename); + void SetIsolinesDir(std::string const & dir); + + void DumpCitiesBoundaries(std::string const & filename); + void DumpRoutingCitiesBoundaries(std::string const & collectorFilename, + std::string const & dumpPath); + + // FinalProcessorIntermediateMwmInterface overrides: + void Process() override; + +private: + void ProcessBooking(); + void ProcessRoutingCityBoundaries(); + void ProcessCities(); + void ProcessCoastline(); + void ProcessRoundabouts(); + void AddFakeNodes(); + void AddIsolines(); + void Finish(); + + std::string m_borderPath; + std::string m_temporaryMwmPath; + std::string m_isolinesPath; + std::string m_citiesAreasTmpFilename; + std::string m_citiesBoundariesFilename; + std::string m_hotelsFilename; + std::string m_coastlineGeomFilename; + std::string m_worldCoastsFilename; + std::string m_citiesFilename; + std::string m_fakeNodesFilename; + std::string m_miniRoundaboutsFilename; + + std::string m_routingCityBoundariesCollectorFilename; + std::string m_routingCityBoundariesDumpPath; + std::string m_hierarchySrcFilename; + + bool m_haveBordersForWholeWorld; + size_t m_threadsCount; +}; +} // namespace generator diff --git a/generator/final_processor_interface.cpp b/generator/final_processor_interface.cpp new file mode 100644 index 0000000000..b34a6ec50c --- /dev/null +++ b/generator/final_processor_interface.cpp @@ -0,0 +1,28 @@ +#include "generator/final_processor_interface.hpp" + +namespace generator +{ +FinalProcessorIntermediateMwmInterface::FinalProcessorIntermediateMwmInterface( + FinalProcessorPriority priority) + : m_priority(priority) +{ +} + +bool FinalProcessorIntermediateMwmInterface::operator<( + FinalProcessorIntermediateMwmInterface const & other) const +{ + return m_priority < other.m_priority; +} + +bool FinalProcessorIntermediateMwmInterface::operator==( + FinalProcessorIntermediateMwmInterface const & other) const +{ + return m_priority == other.m_priority; +} + +bool FinalProcessorIntermediateMwmInterface::operator!=( + FinalProcessorIntermediateMwmInterface const & other) const +{ + return m_priority != other.m_priority; +} +} // namespace generator diff --git a/generator/final_processor_interface.hpp b/generator/final_processor_interface.hpp new file mode 100644 index 0000000000..08e698322a --- /dev/null +++ b/generator/final_processor_interface.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace generator +{ +enum class FinalProcessorPriority : uint8_t +{ + CountriesOrWorld = 1, + WorldCoasts = 2, + Complex = 3 +}; + +// Classes that inherit this interface implement the final stage of intermediate mwm processing. +// For example, attempt to merge the coastline or adding external elements. +// Each derived class has a priority. This is done to comply with the order of processing +// intermediate mwm, taking into account the dependencies between them. For example, before adding a +// coastline to a country, we must build coastline. Processors with higher priority will be called +// first. Processors with the same priority can run in parallel. +class FinalProcessorIntermediateMwmInterface +{ +public: + explicit FinalProcessorIntermediateMwmInterface(FinalProcessorPriority priority); + virtual ~FinalProcessorIntermediateMwmInterface() = default; + + virtual void Process() = 0; + + bool operator<(FinalProcessorIntermediateMwmInterface const & other) const; + bool operator==(FinalProcessorIntermediateMwmInterface const & other) const; + bool operator!=(FinalProcessorIntermediateMwmInterface const & other) const; + +protected: + FinalProcessorPriority m_priority; +}; + +} // namespace generator diff --git a/generator/final_processor_intermediate_mwm.cpp b/generator/final_processor_intermediate_mwm.cpp deleted file mode 100644 index 1fd4133802..0000000000 --- a/generator/final_processor_intermediate_mwm.cpp +++ /dev/null @@ -1,773 +0,0 @@ -#include "generator/final_processor_intermediate_mwm.hpp" - -#include "generator/affiliation.hpp" -#include "generator/booking_dataset.hpp" -#include "generator/complex_loader.hpp" -#include "generator/feature_merger.hpp" -#include "generator/isolines_generator.hpp" -#include "generator/mini_roundabout_transformer.hpp" -#include "generator/node_mixer.hpp" -#include "generator/osm2type.hpp" -#include "generator/place_processor.hpp" -#include "generator/promo_catalog_cities.hpp" -#include "generator/routing_city_boundaries_processor.hpp" -#include "generator/type_helper.hpp" -#include "generator/utils.hpp" - -#include "routing/speed_camera_prohibition.hpp" - -#include "indexer/classificator.hpp" -#include "indexer/feature_algo.hpp" - -#include "platform/platform.hpp" - -#include "base/assert.hpp" -#include "base/file_name_utils.hpp" -#include "base/geo_object_id.hpp" -#include "base/stl_helpers.hpp" -#include "base/string_utils.hpp" -#include "base/thread_pool_computational.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "defines.hpp" - -using namespace base::thread_pool::computational; -using namespace feature; -using namespace serialization_policy; - -namespace generator -{ -namespace -{ -template -void ForEachCountry(std::string const & temporaryMwmPath, ToDo && toDo) -{ - Platform::FilesList fileList; - Platform::GetFilesByExt(temporaryMwmPath, DATA_FILE_EXTENSION_TMP, fileList); - for (auto const & filename : fileList) - toDo(filename); -} - -std::vector> GetAffiliations(std::vector const & fbs, - AffiliationInterface const & affiliation, - size_t threadsCount) -{ - ThreadPool pool(threadsCount); - std::vector>> futuresAffiliations; - for (auto const & fb : fbs) - { - auto result = pool.Submit([&]() { return affiliation.GetAffiliations(fb); }); - futuresAffiliations.emplace_back(std::move(result)); - } - - std::vector> resultAffiliations; - resultAffiliations.reserve(futuresAffiliations.size()); - for (auto & f : futuresAffiliations) - resultAffiliations.emplace_back(f.get()); - - return resultAffiliations; -} - -// Writes |fbs| to countries tmp.mwm files that |fbs| belongs to according to |affiliations|. -template -void AppendToCountries(std::vector const & fbs, - std::vector> const & affiliations, - std::string const & temporaryMwmPath, size_t threadsCount) -{ - std::unordered_map> countryToFbsIndexes; - for (size_t i = 0; i < fbs.size(); ++i) - { - for (auto const & country : affiliations[i]) - countryToFbsIndexes[country].emplace_back(i); - } - - ThreadPool pool(threadsCount); - for (auto && p : countryToFbsIndexes) - { - pool.SubmitWork([&, country{std::move(p.first)}, indexes{std::move(p.second)}]() { - auto const path = base::JoinPath(temporaryMwmPath, country + DATA_FILE_EXTENSION_TMP); - FeatureBuilderWriter collector(path, FileWriter::Op::OP_APPEND); - for (auto const index : indexes) - collector.Write(fbs[index]); - }); - } -} - -void Sort(std::vector & fbs) -{ - std::sort(std::begin(fbs), std::end(fbs), [](auto const & l, auto const & r) { - auto const lGeomType = static_cast(l.GetGeomType()); - auto const rGeomType = static_cast(r.GetGeomType()); - - auto const lId = l.HasOsmIds() ? l.GetMostGenericOsmId() : base::GeoObjectId(); - auto const rId = r.HasOsmIds() ? r.GetMostGenericOsmId() : base::GeoObjectId(); - - auto const lPointsCount = l.GetPointsCount(); - auto const rPointsCount = r.GetPointsCount(); - - auto const lKeyPoint = l.GetKeyPoint(); - auto const rKeyPoint = r.GetKeyPoint(); - - return std::tie(lGeomType, lId, lPointsCount, lKeyPoint) < - std::tie(rGeomType, rId, rPointsCount, rKeyPoint); - }); -} - -std::string GetCountryNameFromTmpMwmPath(std::string filename) -{ - strings::ReplaceLast(filename, DATA_FILE_EXTENSION_TMP, ""); - return filename; -} - -bool FilenameIsCountry(std::string const & filename, AffiliationInterface const & affiliation) -{ - return affiliation.HasCountryByName(GetCountryNameFromTmpMwmPath(filename)); -} - -class PlaceHelper -{ -public: - PlaceHelper() : m_table(std::make_shared()), m_processor(m_table) {} - - explicit PlaceHelper(std::string const & filename) : PlaceHelper() - { - ForEachFromDatRawFormat( - filename, [&](auto const & fb, auto const &) { m_processor.Add(fb); }); - } - - static bool IsPlace(FeatureBuilder const & fb) - { - auto const type = GetPlaceType(fb); - return type != ftype::GetEmptyValue() && !fb.GetName().empty() && NeedProcessPlace(fb); - } - - bool Process(FeatureBuilder const & fb) - { - m_processor.Add(fb); - return true; - } - - std::vector GetFeatures() { return m_processor.ProcessPlaces(); } - - std::shared_ptr GetTable() const { return m_table; } - -private: - std::shared_ptr m_table; - PlaceProcessor m_processor; -}; - -class ProcessorCities -{ -public: - ProcessorCities(std::string const & temporaryMwmPath, AffiliationInterface const & affiliation, - PlaceHelper & citiesHelper, size_t threadsCount = 1) - : m_temporaryMwmPath(temporaryMwmPath) - , m_affiliation(affiliation) - , m_citiesHelper(citiesHelper) - , m_threadsCount(threadsCount) - { - } - - void SetPromoCatalog(std::string const & filename) { m_citiesFilename = filename; } - - void Process() - { - std::vector>> citiesResults; - ThreadPool pool(m_threadsCount); - ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { - auto cities = pool.Submit([&, filename]() { - std::vector cities; - if (!FilenameIsCountry(filename, m_affiliation)) - return cities; - - auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); - auto fbs = ReadAllDatRawFormat(fullPath); - FeatureBuilderWriter writer(fullPath); - for (size_t i = 0; i < fbs.size(); ++i) - { - if (PlaceHelper::IsPlace(fbs[i])) - cities.emplace_back(std::move(fbs[i])); - else - writer.Write(std::move(fbs[i])); - } - - return cities; - }); - citiesResults.emplace_back(std::move(cities)); - }); - - for (auto & v : citiesResults) - { - auto const cities = v.get(); - for (auto const & city : cities) - m_citiesHelper.Process(city); - } - auto fbsWithIds = m_citiesHelper.GetFeatures(); - if (!m_citiesFilename.empty()) - ProcessForPromoCatalog(fbsWithIds); - - std::vector fbs; - fbs.reserve(fbsWithIds.size()); - base::Transform(fbsWithIds, std::back_inserter(fbs), base::RetrieveFirst()); - auto const affiliations = GetAffiliations(fbs, m_affiliation, m_threadsCount); - AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); - } - -private: - void ProcessForPromoCatalog(std::vector & fbs) - { - auto const cities = promo::LoadCities(m_citiesFilename); - for (auto & fbWithIds : fbs) - { - for (auto const & id : fbWithIds.second) - { - if (cities.count(id) == 0) - continue; - - auto static const kPromoType = classif().GetTypeByPath({"sponsored", "promo_catalog"}); - FeatureParams & params = fbWithIds.first.GetParams(); - params.AddType(kPromoType); - } - } - } - - std::string m_citiesFilename; - std::string m_temporaryMwmPath; - AffiliationInterface const & m_affiliation; - PlaceHelper & m_citiesHelper; - size_t m_threadsCount; -}; -} // namespace - -FinalProcessorIntermediateMwmInterface::FinalProcessorIntermediateMwmInterface( - FinalProcessorPriority priority) - : m_priority(priority) -{ -} - -bool FinalProcessorIntermediateMwmInterface::operator<( - FinalProcessorIntermediateMwmInterface const & other) const -{ - return m_priority < other.m_priority; -} - -bool FinalProcessorIntermediateMwmInterface::operator==( - FinalProcessorIntermediateMwmInterface const & other) const -{ - return !(*this < other || other < *this); -} - -bool FinalProcessorIntermediateMwmInterface::operator!=( - FinalProcessorIntermediateMwmInterface const & other) const -{ - return !(*this == other); -} - -CountryFinalProcessor::CountryFinalProcessor(std::string const & borderPath, - std::string const & temporaryMwmPath, - bool haveBordersForWholeWorld, size_t threadsCount) - : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::CountriesOrWorld) - , m_borderPath(borderPath) - , m_temporaryMwmPath(temporaryMwmPath) - , m_haveBordersForWholeWorld(haveBordersForWholeWorld) - , m_threadsCount(threadsCount) -{ -} - -void CountryFinalProcessor::SetBooking(std::string const & filename) -{ - m_hotelsFilename = filename; -} - -void CountryFinalProcessor::SetCitiesAreas(std::string const & filename) -{ - m_citiesAreasTmpFilename = filename; -} - -void CountryFinalProcessor::SetPromoCatalog(std::string const & filename) -{ - m_citiesFilename = filename; -} - -void CountryFinalProcessor::DumpCitiesBoundaries(std::string const & filename) -{ - m_citiesBoundariesFilename = filename; -} - -void CountryFinalProcessor::DumpRoutingCitiesBoundaries(std::string const & collectorFilename, - std::string const & dumpPath) -{ - m_routingCityBoundariesCollectorFilename = collectorFilename; - m_routingCityBoundariesDumpPath = dumpPath; -} - -void CountryFinalProcessor::SetCoastlines(std::string const & coastlineGeomFilename, - std::string const & worldCoastsFilename) -{ - m_coastlineGeomFilename = coastlineGeomFilename; - m_worldCoastsFilename = worldCoastsFilename; -} - -void CountryFinalProcessor::SetFakeNodes(std::string const & filename) -{ - m_fakeNodesFilename = filename; -} - -void CountryFinalProcessor::SetMiniRoundabouts(std::string const & filename) -{ - m_miniRoundaboutsFilename = filename; -} - -void CountryFinalProcessor::SetIsolinesDir(std::string const & dir) -{ - m_isolinesPath = dir; -} - -void CountryFinalProcessor::Process() -{ - if (!m_hotelsFilename.empty()) - ProcessBooking(); - if (!m_routingCityBoundariesCollectorFilename.empty()) - ProcessRoutingCityBoundaries(); - if (!m_citiesAreasTmpFilename.empty() || !m_citiesFilename.empty()) - ProcessCities(); - if (!m_coastlineGeomFilename.empty()) - ProcessCoastline(); - if (!m_miniRoundaboutsFilename.empty()) - ProcessRoundabouts(); - if (!m_fakeNodesFilename.empty()) - AddFakeNodes(); - if (!m_isolinesPath.empty()) - AddIsolines(); - Finish(); -} - -void CountryFinalProcessor::ProcessBooking() -{ - BookingDataset dataset(m_hotelsFilename); - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - { - ThreadPool pool(m_threadsCount); - ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { - pool.SubmitWork([&, filename]() { - std::vector cities; - if (!FilenameIsCountry(filename, affiliation)) - return; - - auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); - auto fbs = ReadAllDatRawFormat(fullPath); - FeatureBuilderWriter writer(fullPath); - for (auto & fb : fbs) - { - auto const id = dataset.FindMatchingObjectId(fb); - if (id == BookingHotel::InvalidObjectId()) - { - writer.Write(fb); - } - else - { - dataset.PreprocessMatchedOsmObject(id, fb, [&](FeatureBuilder & newFeature) { - if (newFeature.PreSerialize()) - writer.Write(newFeature); - }); - } - } - }); - }); - } - std::vector fbs; - dataset.BuildOsmObjects([&](auto && fb) { fbs.emplace_back(std::move(fb)); }); - auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); - AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); -} - -void CountryFinalProcessor::ProcessRoundabouts() -{ - MiniRoundaboutTransformer helper(m_miniRoundaboutsFilename); - - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - { - ThreadPool pool(m_threadsCount); - ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { - pool.SubmitWork([&, filename]() { - if (!FilenameIsCountry(filename, affiliation)) - return; - - auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); - auto fbs = ReadAllDatRawFormat(fullPath); - FeatureBuilderWriter writer(fullPath); - - // Adds new way features generated from mini-roundabout nodes with those nodes ids. - // Transforms points on roads to connect them with these new roundabout junctions. - helper.ProcessRoundabouts(affiliation, fbs); - for (auto const & fb : fbs) - writer.Write(fb); - }); - }); - } -} - -void CountryFinalProcessor::AddIsolines() -{ - // For generated isolines must be built isolines_info section based on the same - // binary isolines file. - IsolineFeaturesGenerator isolineFeaturesGenerator(m_isolinesPath); - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - ThreadPool pool(m_threadsCount); - ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { - pool.SubmitWork([&, filename]() { - if (!FilenameIsCountry(filename, affiliation)) - return; - auto const countryName = GetCountryNameFromTmpMwmPath(filename); - - auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); - FeatureBuilderWriter writer(fullPath, FileWriter::Op::OP_APPEND); - isolineFeaturesGenerator.GenerateIsolines( - countryName, [&writer](feature::FeatureBuilder && fb){ writer.Write(fb); }); - }); - }); -} - -void CountryFinalProcessor::ProcessRoutingCityBoundaries() -{ - CHECK( - !m_routingCityBoundariesCollectorFilename.empty() && !m_routingCityBoundariesDumpPath.empty(), - ()); - - RoutingCityBoundariesProcessor processor(m_routingCityBoundariesCollectorFilename, - m_routingCityBoundariesDumpPath); - processor.ProcessDataFromCollector(); -} - -void CountryFinalProcessor::ProcessCities() -{ - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - auto citiesHelper = - m_citiesAreasTmpFilename.empty() ? PlaceHelper() : PlaceHelper(m_citiesAreasTmpFilename); - ProcessorCities processorCities(m_temporaryMwmPath, affiliation, citiesHelper, m_threadsCount); - processorCities.SetPromoCatalog(m_citiesFilename); - processorCities.Process(); - if (m_citiesBoundariesFilename.empty()) - return; - - auto const citiesTable = citiesHelper.GetTable(); - LOG(LINFO, ("Dumping cities boundaries to", m_citiesBoundariesFilename)); - SerializeBoundariesTable(m_citiesBoundariesFilename, *citiesTable); -} - -void CountryFinalProcessor::ProcessCoastline() -{ - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - auto fbs = ReadAllDatRawFormat(m_coastlineGeomFilename); - auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); - AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); - FeatureBuilderWriter<> collector(m_worldCoastsFilename); - for (size_t i = 0; i < fbs.size(); ++i) - { - fbs[i].AddName("default", strings::JoinStrings(affiliations[i], ';')); - collector.Write(fbs[i]); - } -} - -void CountryFinalProcessor::AddFakeNodes() -{ - std::vector fbs; - MixFakeNodes(m_fakeNodesFilename, [&](auto & element) { - FeatureBuilder fb; - fb.SetCenter(mercator::FromLatLon(element.m_lat, element.m_lon)); - fb.SetOsmId(base::MakeOsmNode(element.m_id)); - ftype::GetNameAndType(&element, fb.GetParams()); - fbs.emplace_back(std::move(fb)); - }); - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - auto const affiliations = GetAffiliations(fbs, affiliation, m_threadsCount); - AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); -} - -void CountryFinalProcessor::Finish() -{ - auto const affiliation = CountriesFilesIndexAffiliation(m_borderPath, m_haveBordersForWholeWorld); - ThreadPool pool(m_threadsCount); - ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { - pool.SubmitWork([&, filename]() { - if (!FilenameIsCountry(filename, affiliation)) - return; - - auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); - auto fbs = ReadAllDatRawFormat(fullPath); - Sort(fbs); - - auto const speedCamerasProhibited = - routing::AreSpeedCamerasProhibited(platform::CountryFile(filename)); - static auto const speedCameraType = classif().GetTypeByPath({"highway", "speed_camera"}); - - FeatureBuilderWriter<> collector(fullPath); - for (auto & fb : fbs) - { - // Removing point features with speed cameras type from geometry index for some countries. - if (speedCamerasProhibited && fb.IsPoint() && fb.HasType(speedCameraType)) - continue; - - collector.Write(fb); - } - }); - }); -} - -WorldFinalProcessor::WorldFinalProcessor(std::string const & temporaryMwmPath, - std::string const & coastlineGeomFilename) - : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::CountriesOrWorld) - , m_temporaryMwmPath(temporaryMwmPath) - , m_worldTmpFilename(base::JoinPath(m_temporaryMwmPath, WORLD_FILE_NAME) + - DATA_FILE_EXTENSION_TMP) - , m_coastlineGeomFilename(coastlineGeomFilename) -{ -} - -void WorldFinalProcessor::SetPopularPlaces(std::string const & filename) -{ - m_popularPlacesFilename = filename; -} - -void WorldFinalProcessor::SetCitiesAreas(std::string const & filename) -{ - m_citiesAreasTmpFilename = filename; -} - -void WorldFinalProcessor::SetPromoCatalog(std::string const & filename) -{ - m_citiesFilename = filename; -} - -void WorldFinalProcessor::Process() -{ - if (!m_citiesAreasTmpFilename.empty() || !m_citiesFilename.empty()) - ProcessCities(); - - auto fbs = ReadAllDatRawFormat(m_worldTmpFilename); - Sort(fbs); - WorldGenerator generator(m_worldTmpFilename, m_coastlineGeomFilename, m_popularPlacesFilename); - for (auto & fb : fbs) - generator.Process(fb); - - generator.DoMerge(); -} - -void WorldFinalProcessor::ProcessCities() -{ - auto const affiliation = SingleAffiliation(WORLD_FILE_NAME); - auto citiesHelper = - m_citiesAreasTmpFilename.empty() ? PlaceHelper() : PlaceHelper(m_citiesAreasTmpFilename); - ProcessorCities processorCities(m_temporaryMwmPath, affiliation, citiesHelper); - processorCities.SetPromoCatalog(m_citiesFilename); - processorCities.Process(); -} - -CoastlineFinalProcessor::CoastlineFinalProcessor(std::string const & filename) - : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::WorldCoasts) - , m_filename(filename) -{ -} - -void CoastlineFinalProcessor::SetCoastlinesFilenames(std::string const & geomFilename, - std::string const & rawGeomFilename) -{ - m_coastlineGeomFilename = geomFilename; - m_coastlineRawGeomFilename = rawGeomFilename; -} - -void CoastlineFinalProcessor::Process() -{ - auto fbs = ReadAllDatRawFormat(m_filename); - Sort(fbs); - for (auto && fb : fbs) - m_generator.Process(std::move(fb)); - - FeaturesAndRawGeometryCollector collector(m_coastlineGeomFilename, m_coastlineRawGeomFilename); - // Check and stop if some coasts were not merged. - CHECK(m_generator.Finish(), ()); - LOG(LINFO, ("Generating coastline polygons.")); - size_t totalFeatures = 0; - size_t totalPoints = 0; - size_t totalPolygons = 0; - std::vector outputFbs; - m_generator.GetFeatures(outputFbs); - for (auto & fb : outputFbs) - { - collector.Collect(fb); - ++totalFeatures; - totalPoints += fb.GetPointsCount(); - totalPolygons += fb.GetPolygonsCount(); - } - - LOG(LINFO, ("Total features:", totalFeatures, "total polygons:", totalPolygons, - "total points:", totalPoints)); -} - -ComplexFinalProcessor::ComplexFinalProcessor(std::string const & mwmTmpPath, - std::string const & outFilename, size_t threadsCount) - : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::Complex) - , m_mwmTmpPath(mwmTmpPath) - , m_outFilename(outFilename) - , m_threadsCount(threadsCount) -{ -} - -void ComplexFinalProcessor::SetGetMainTypeFunction(hierarchy::GetMainTypeFn const & getMainType) -{ - m_getMainType = getMainType; -} - -void ComplexFinalProcessor::SetFilter(std::shared_ptr const & filter) -{ - m_filter = filter; -} - -void ComplexFinalProcessor::SetGetNameFunction(hierarchy::GetNameFn const & getName) -{ - m_getName = getName; -} - -void ComplexFinalProcessor::SetPrintFunction(hierarchy::PrintFn const & printFunction) -{ - m_printFunction = printFunction; -} - -void ComplexFinalProcessor::UseCentersEnricher(std::string const & mwmPath, - std::string const & osm2ftPath) -{ - m_useCentersEnricher = true; - m_mwmPath = mwmPath; - m_osm2ftPath = osm2ftPath; -} - -std::unique_ptr ComplexFinalProcessor::CreateEnricher( - std::string const & countryName) const -{ - return std::make_unique( - base::JoinPath(m_osm2ftPath, countryName + DATA_FILE_EXTENSION OSM2FEATURE_FILE_EXTENSION), - base::JoinPath(m_mwmPath, countryName + DATA_FILE_EXTENSION)); -} - -void ComplexFinalProcessor::UseBuildingPartsInfo(std::string const & filename) -{ - m_buildingPartsFilename = filename; -} - -void ComplexFinalProcessor::Process() -{ - if (!m_buildingPartsFilename.empty()) - m_buildingToParts = std::make_unique(m_buildingPartsFilename); - - ThreadPool pool(m_threadsCount); - std::vector>> futures; - ForEachCountry(m_mwmTmpPath, [&](auto const & filename) { - auto future = pool.Submit([&, filename]() { - auto countryName = GetCountryNameFromTmpMwmPath(filename); - // https://wiki.openstreetmap.org/wiki/Simple_3D_buildings - // An object with tag 'building:part' is a part of a relation with outline 'building' or - // is contained in an object with tag 'building'. We will split data and work with - // these cases separately. First of all let's remove objects with tag building:part is contained - // in relations. We will add them back after data processing. - std::unordered_map relationBuildingParts; - - auto fbs = ReadAllDatRawFormat( - base::JoinPath(m_mwmTmpPath, filename)); - if (m_buildingToParts) - relationBuildingParts = RemoveRelationBuildingParts(fbs); - - // This case is second. We build a hierarchy using the geometry of objects and their nesting. - auto trees = hierarchy::BuildHierarchy(std::move(fbs), m_getMainType, m_filter); - - // We remove tree roots with tag 'building:part'. - base::EraseIf(trees, [](auto const & node) { - static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance(); - return buildingPartChecker(node->GetData().GetTypes()); - }); - - // We want to transform - // building - // |_building-part - // |_building-part - // to - // building - // |_building-part - // |_building-part - hierarchy::FlattenBuildingParts(trees); - // In the end we add objects, which were saved by the collector. - if (m_buildingToParts) - { - hierarchy::AddChildrenTo(trees, [&](auto const & compositeId) { - auto const & ids = m_buildingToParts->GetBuildingPartsByOutlineId(compositeId); - std::vector places; - places.reserve(ids.size()); - for (auto const & id : ids) - { - if (relationBuildingParts.count(id) == 0) - continue; - - places.emplace_back(hierarchy::HierarchyPlace(relationBuildingParts[id])); - } - return places; - }); - } - - // We create and save hierarchy lines. - hierarchy::HierarchyLinesBuilder hierarchyBuilder(std::move(trees)); - if (m_useCentersEnricher) - hierarchyBuilder.SetHierarchyEntryEnricher(CreateEnricher(countryName)); - - hierarchyBuilder.SetCountry(countryName); - hierarchyBuilder.SetGetMainTypeFunction(m_getMainType); - hierarchyBuilder.SetGetNameFunction(m_getName); - return hierarchyBuilder.GetHierarchyLines(); - }); - futures.emplace_back(std::move(future)); - }); - std::vector allLines; - for (auto & f : futures) - { - auto const lines = f.get(); - allLines.insert(std::cend(allLines), std::cbegin(lines), std::cend(lines)); - } - WriteLines(allLines); -} - -std::unordered_map -ComplexFinalProcessor::RemoveRelationBuildingParts(std::vector & fbs) -{ - CHECK(m_buildingToParts, ()); - - auto it = std::partition(std::begin(fbs), std::end(fbs), [&](auto const & fb) { - return !m_buildingToParts->HasBuildingPart(fb.GetMostGenericOsmId()); - }); - - std::unordered_map buildingParts; - buildingParts.reserve(static_cast(std::distance(it, std::end(fbs)))); - - std::transform(it, std::end(fbs), std::inserter(buildingParts, std::begin(buildingParts)), [](auto && fb) { - auto const id = fb.GetMostGenericOsmId(); - return std::make_pair(id, std::move(fb)); - }); - - fbs.resize(static_cast(std::distance(std::begin(fbs), it))); - return buildingParts; -} - -void ComplexFinalProcessor::WriteLines(std::vector const & lines) -{ - std::ofstream stream; - stream.exceptions(std::fstream::failbit | std::fstream::badbit); - stream.open(m_outFilename); - for (auto const & line : lines) - stream << m_printFunction(line) << '\n'; -} -} // namespace generator diff --git a/generator/final_processor_intermediate_mwm.hpp b/generator/final_processor_intermediate_mwm.hpp deleted file mode 100644 index c6f1edfb22..0000000000 --- a/generator/final_processor_intermediate_mwm.hpp +++ /dev/null @@ -1,184 +0,0 @@ -#pragma once - -#include "generator/coastlines_generator.hpp" -#include "generator/collector_building_parts.hpp" -#include "generator/feature_builder.hpp" -#include "generator/feature_generator.hpp" -#include "generator/filter_interface.hpp" -#include "generator/hierarchy.hpp" -#include "generator/hierarchy_entry.hpp" -#include "generator/world_map_generator.hpp" - -#include -#include -#include - -namespace generator -{ -enum class FinalProcessorPriority : uint8_t -{ - CountriesOrWorld = 1, - WorldCoasts = 2, - Complex = 3 -}; - -// Classes that inherit this interface implement the final stage of intermediate mwm processing. -// For example, attempt to merge the coastline or adding external elements. -// Each derived class has a priority. This is done to comply with the order of processing -// intermediate mwm, taking into account the dependencies between them. For example, before adding a -// coastline to a country, we must build coastline. Processors with higher priority will be called -// first. Processors with the same priority can run in parallel. -class FinalProcessorIntermediateMwmInterface -{ -public: - explicit FinalProcessorIntermediateMwmInterface(FinalProcessorPriority priority); - virtual ~FinalProcessorIntermediateMwmInterface() = default; - - virtual void Process() = 0; - - bool operator<(FinalProcessorIntermediateMwmInterface const & other) const; - bool operator==(FinalProcessorIntermediateMwmInterface const & other) const; - bool operator!=(FinalProcessorIntermediateMwmInterface const & other) const; - -protected: - FinalProcessorPriority m_priority; -}; - -class CountryFinalProcessor : public FinalProcessorIntermediateMwmInterface -{ -public: - CountryFinalProcessor(std::string const & borderPath, - std::string const & temporaryMwmPath, - bool haveBordersForWholeWorld, size_t threadsCount); - - void SetBooking(std::string const & filename); - void SetCitiesAreas(std::string const & filename); - void SetPromoCatalog(std::string const & filename); - void SetCoastlines(std::string const & coastlineGeomFilename, - std::string const & worldCoastsFilename); - void SetFakeNodes(std::string const & filename); - void SetMiniRoundabouts(std::string const & filename); - void SetIsolinesDir(std::string const & dir); - - void DumpCitiesBoundaries(std::string const & filename); - void DumpRoutingCitiesBoundaries(std::string const & collectorFilename, - std::string const & dumpPath); - - // FinalProcessorIntermediateMwmInterface overrides: - void Process() override; - -private: - void ProcessBooking(); - void ProcessRoutingCityBoundaries(); - void ProcessCities(); - void ProcessCoastline(); - void ProcessRoundabouts(); - void AddFakeNodes(); - void AddIsolines(); - void Finish(); - - std::string m_borderPath; - std::string m_temporaryMwmPath; - std::string m_isolinesPath; - std::string m_citiesAreasTmpFilename; - std::string m_citiesBoundariesFilename; - std::string m_hotelsFilename; - std::string m_coastlineGeomFilename; - std::string m_worldCoastsFilename; - std::string m_citiesFilename; - std::string m_fakeNodesFilename; - std::string m_miniRoundaboutsFilename; - - std::string m_routingCityBoundariesCollectorFilename; - std::string m_routingCityBoundariesDumpPath; - std::string m_hierarchySrcFilename; - - bool m_haveBordersForWholeWorld; - size_t m_threadsCount; -}; - -class WorldFinalProcessor : public FinalProcessorIntermediateMwmInterface -{ -public: - using WorldGenerator = WorldMapGenerator; - - explicit WorldFinalProcessor(std::string const & temporaryMwmPath, - std::string const & coastlineGeomFilename); - - void SetPopularPlaces(std::string const & filename); - void SetCitiesAreas(std::string const & filename); - void SetPromoCatalog(std::string const & filename); - - // FinalProcessorIntermediateMwmInterface overrides: - void Process() override; - -private: - void ProcessCities(); - - std::string m_temporaryMwmPath; - std::string m_worldTmpFilename; - std::string m_coastlineGeomFilename; - std::string m_popularPlacesFilename; - std::string m_citiesAreasTmpFilename; - std::string m_citiesFilename; -}; - -class CoastlineFinalProcessor : public FinalProcessorIntermediateMwmInterface -{ -public: - explicit CoastlineFinalProcessor(std::string const & filename); - - void SetCoastlinesFilenames(std::string const & geomFilename, - std::string const & rawGeomFilename); - - // FinalProcessorIntermediateMwmInterface overrides: - void Process() override; - -private: - std::string m_filename; - std::string m_coastlineGeomFilename; - std::string m_coastlineRawGeomFilename; - CoastlineFeaturesGenerator m_generator; -}; - -// Class ComplexFinalProcessor generates hierarchies for each previously filtered mwm.tmp file. -// Warning: If the border separates the complex, then a situation is possible in which two logically -// identical complexes are generated, but with different representations. -class ComplexFinalProcessor : public FinalProcessorIntermediateMwmInterface -{ -public: - ComplexFinalProcessor(std::string const & mwmTmpPath, std::string const & outFilename, - size_t threadsCount); - - void SetGetMainTypeFunction(hierarchy::GetMainTypeFn const & getMainType); - void SetFilter(std::shared_ptr const & filter); - void SetGetNameFunction(hierarchy::GetNameFn const & getName); - void SetPrintFunction(hierarchy::PrintFn const & printFunction); - - void UseCentersEnricher(std::string const & mwmPath, std::string const & osm2ftPath); - void UseBuildingPartsInfo(std::string const & filename); - - // FinalProcessorIntermediateMwmInterface overrides: - void Process() override; - -private: - std::unique_ptr CreateEnricher( - std::string const & countryName) const; - void WriteLines(std::vector const & lines); - std::unordered_map RemoveRelationBuildingParts( - std::vector & fbs); - - hierarchy::GetMainTypeFn m_getMainType; - hierarchy::PrintFn m_printFunction; - hierarchy::GetNameFn m_getName; - std::shared_ptr m_filter; - std::unique_ptr m_buildingToParts; - bool m_useCentersEnricher = false; - std::string m_mwmTmpPath; - std::string m_outFilename; - std::string m_mwmPath; - std::string m_osm2ftPath; - std::string m_buildingPartsFilename; - size_t m_threadsCount; -}; -} // namespace generator diff --git a/generator/final_processor_utils.cpp b/generator/final_processor_utils.cpp new file mode 100644 index 0000000000..76fbacfa23 --- /dev/null +++ b/generator/final_processor_utils.cpp @@ -0,0 +1,141 @@ +#include "generator/final_processor_utils.hpp" + +#include "generator/promo_catalog_cities.hpp" + +#include "indexer/feature_data.hpp" + +#include "base/stl_helpers.hpp" +#include "base/string_utils.hpp" + +#include +#include +#include + +using namespace base::thread_pool::computational; +using namespace feature; + +namespace generator +{ +ProcessorCities::ProcessorCities(std::string const & temporaryMwmPath, + AffiliationInterface const & affiliation, + PlaceHelper & citiesHelper, size_t threadsCount) + : m_temporaryMwmPath(temporaryMwmPath) + , m_affiliation(affiliation) + , m_citiesHelper(citiesHelper) + , m_threadsCount(threadsCount) +{ +} + +void ProcessorCities::SetPromoCatalog(std::string const & filename) { m_citiesFilename = filename; } + +void ProcessorCities::Process() +{ + std::vector>> citiesResults; + ThreadPool pool(m_threadsCount); + ForEachCountry(m_temporaryMwmPath, [&](auto const & filename) { + auto cities = pool.Submit([&, filename]() { + std::vector cities; + if (!FilenameIsCountry(filename, m_affiliation)) + return cities; + + auto const fullPath = base::JoinPath(m_temporaryMwmPath, filename); + auto fbs = ReadAllDatRawFormat(fullPath); + FeatureBuilderWriter writer(fullPath); + for (size_t i = 0; i < fbs.size(); ++i) + { + if (PlaceHelper::IsPlace(fbs[i])) + cities.emplace_back(std::move(fbs[i])); + else + writer.Write(std::move(fbs[i])); + } + + return cities; + }); + citiesResults.emplace_back(std::move(cities)); + }); + + for (auto & v : citiesResults) + { + auto const cities = v.get(); + for (auto const & city : cities) + m_citiesHelper.Process(city); + } + auto fbsWithIds = m_citiesHelper.GetFeatures(); + if (!m_citiesFilename.empty()) + ProcessForPromoCatalog(fbsWithIds); + + std::vector fbs; + fbs.reserve(fbsWithIds.size()); + base::Transform(fbsWithIds, std::back_inserter(fbs), base::RetrieveFirst()); + auto const affiliations = GetAffiliations(fbs, m_affiliation, m_threadsCount); + AppendToCountries(fbs, affiliations, m_temporaryMwmPath, m_threadsCount); +} + +void ProcessorCities::ProcessForPromoCatalog(std::vector & fbs) +{ + auto const cities = promo::LoadCities(m_citiesFilename); + for (auto & fbWithIds : fbs) + { + for (auto const & id : fbWithIds.second) + { + if (cities.count(id) == 0) + continue; + + auto static const kPromoType = classif().GetTypeByPath({"sponsored", "promo_catalog"}); + FeatureParams & params = fbWithIds.first.GetParams(); + params.AddType(kPromoType); + } + } +} + +void Sort(std::vector & fbs) +{ + std::sort(std::begin(fbs), std::end(fbs), [](auto const & l, auto const & r) { + auto const lGeomType = static_cast(l.GetGeomType()); + auto const rGeomType = static_cast(r.GetGeomType()); + + auto const lId = l.HasOsmIds() ? l.GetMostGenericOsmId() : base::GeoObjectId(); + auto const rId = r.HasOsmIds() ? r.GetMostGenericOsmId() : base::GeoObjectId(); + + auto const lPointsCount = l.GetPointsCount(); + auto const rPointsCount = r.GetPointsCount(); + + auto const lKeyPoint = l.GetKeyPoint(); + auto const rKeyPoint = r.GetKeyPoint(); + + return std::tie(lGeomType, lId, lPointsCount, lKeyPoint) < + std::tie(rGeomType, rId, rPointsCount, rKeyPoint); + }); +} + +std::vector> GetAffiliations(std::vector const & fbs, + AffiliationInterface const & affiliation, + size_t threadsCount) +{ + ThreadPool pool(threadsCount); + std::vector>> futuresAffiliations; + for (auto const & fb : fbs) + { + auto result = pool.Submit([&]() { return affiliation.GetAffiliations(fb); }); + futuresAffiliations.emplace_back(std::move(result)); + } + + std::vector> resultAffiliations; + resultAffiliations.reserve(futuresAffiliations.size()); + for (auto & f : futuresAffiliations) + resultAffiliations.emplace_back(f.get()); + + return resultAffiliations; +} + +std::string GetCountryNameFromTmpMwmPath(std::string filename) +{ + strings::ReplaceLast(filename, DATA_FILE_EXTENSION_TMP, ""); + return filename; +} + +bool FilenameIsCountry(std::string const & filename, AffiliationInterface const & affiliation) +{ + return affiliation.HasCountryByName(GetCountryNameFromTmpMwmPath(filename)); +} +} // namespace generator diff --git a/generator/final_processor_utils.hpp b/generator/final_processor_utils.hpp new file mode 100644 index 0000000000..38ebdba0e3 --- /dev/null +++ b/generator/final_processor_utils.hpp @@ -0,0 +1,123 @@ +#pragma once + +#include "generator/affiliation.hpp" +#include "generator/cities_boundaries_builder.hpp" +#include "generator/feature_builder.hpp" +#include "generator/place_processor.hpp" +#include "generator/type_helper.hpp" + +#include "indexer/classificator.hpp" + +#include "platform/platform.hpp" + +#include "base/file_name_utils.hpp" +#include "base/thread_pool_computational.hpp" + +#include "defines.hpp" + +#include +#include +#include +#include +#include + +namespace generator +{ +class PlaceHelper +{ +public: + PlaceHelper() : m_table(std::make_shared()), m_processor(m_table) {} + + explicit PlaceHelper(std::string const & filename) : PlaceHelper() + { + feature::ForEachFromDatRawFormat( + filename, [&](auto const & fb, auto const &) { m_processor.Add(fb); }); + } + + static bool IsPlace(feature::FeatureBuilder const & fb) + { + auto const type = GetPlaceType(fb); + return type != ftype::GetEmptyValue() && !fb.GetName().empty() && NeedProcessPlace(fb); + } + + bool Process(feature::FeatureBuilder const & fb) + { + m_processor.Add(fb); + return true; + } + + std::vector GetFeatures() { return m_processor.ProcessPlaces(); } + + std::shared_ptr GetTable() const { return m_table; } + +private: + std::shared_ptr m_table; + PlaceProcessor m_processor; +}; + +class ProcessorCities +{ +public: + ProcessorCities(std::string const & temporaryMwmPath, + feature::AffiliationInterface const & affiliation, PlaceHelper & citiesHelper, + size_t threadsCount = 1); + + void SetPromoCatalog(std::string const & filename); + + void Process(); + +private: + void ProcessForPromoCatalog(std::vector & fbs); + + std::string m_citiesFilename; + std::string m_temporaryMwmPath; + feature::AffiliationInterface const & m_affiliation; + PlaceHelper & m_citiesHelper; + size_t m_threadsCount; +}; + +template +void ForEachCountry(std::string const & temporaryMwmPath, ToDo && toDo) +{ + Platform::FilesList fileList; + Platform::GetFilesByExt(temporaryMwmPath, DATA_FILE_EXTENSION_TMP, fileList); + for (auto const & filename : fileList) + toDo(filename); +} + +// Writes |fbs| to countries tmp.mwm files that |fbs| belongs to according to |affiliations|. +template +void AppendToCountries(std::vector const & fbs, + std::vector> const & affiliations, + std::string const & temporaryMwmPath, size_t threadsCount) +{ + std::unordered_map> countryToFbsIndexes; + for (size_t i = 0; i < fbs.size(); ++i) + { + for (auto const & country : affiliations[i]) + countryToFbsIndexes[country].emplace_back(i); + } + + base::thread_pool::computational::ThreadPool pool(threadsCount); + for (auto && p : countryToFbsIndexes) + { + pool.SubmitWork([&, country{std::move(p.first)}, indexes{std::move(p.second)}]() { + auto const path = base::JoinPath(temporaryMwmPath, country + DATA_FILE_EXTENSION_TMP); + feature::FeatureBuilderWriter collector(path, FileWriter::Op::OP_APPEND); + for (auto const index : indexes) + collector.Write(fbs[index]); + }); + } +} + +void Sort(std::vector & fbs); + +std::vector> GetAffiliations( + std::vector const & fbs, + feature::AffiliationInterface const & affiliation, size_t threadsCount); + +std::string GetCountryNameFromTmpMwmPath(std::string filename); + +bool FilenameIsCountry(std::string const & filename, + feature::AffiliationInterface const & affiliation); +} // namespace generator diff --git a/generator/final_processor_world.cpp b/generator/final_processor_world.cpp new file mode 100644 index 0000000000..fba4b79bf8 --- /dev/null +++ b/generator/final_processor_world.cpp @@ -0,0 +1,61 @@ +#include "generator/final_processor_world.hpp" + +#include "generator/affiliation.hpp" +#include "generator/feature_builder.hpp" +#include "generator/final_processor_utils.hpp" + +#include "defines.hpp" + +using namespace feature; + +namespace generator +{ +WorldFinalProcessor::WorldFinalProcessor(std::string const & temporaryMwmPath, + std::string const & coastlineGeomFilename) + : FinalProcessorIntermediateMwmInterface(FinalProcessorPriority::CountriesOrWorld) + , m_temporaryMwmPath(temporaryMwmPath) + , m_worldTmpFilename(base::JoinPath(m_temporaryMwmPath, WORLD_FILE_NAME) + + DATA_FILE_EXTENSION_TMP) + , m_coastlineGeomFilename(coastlineGeomFilename) +{ +} + +void WorldFinalProcessor::SetPopularPlaces(std::string const & filename) +{ + m_popularPlacesFilename = filename; +} + +void WorldFinalProcessor::SetCitiesAreas(std::string const & filename) +{ + m_citiesAreasTmpFilename = filename; +} + +void WorldFinalProcessor::SetPromoCatalog(std::string const & filename) +{ + m_citiesFilename = filename; +} + +void WorldFinalProcessor::Process() +{ + if (!m_citiesAreasTmpFilename.empty() || !m_citiesFilename.empty()) + ProcessCities(); + + auto fbs = ReadAllDatRawFormat(m_worldTmpFilename); + Sort(fbs); + WorldGenerator generator(m_worldTmpFilename, m_coastlineGeomFilename, m_popularPlacesFilename); + for (auto & fb : fbs) + generator.Process(fb); + + generator.DoMerge(); +} + +void WorldFinalProcessor::ProcessCities() +{ + auto const affiliation = SingleAffiliation(WORLD_FILE_NAME); + auto citiesHelper = + m_citiesAreasTmpFilename.empty() ? PlaceHelper() : PlaceHelper(m_citiesAreasTmpFilename); + ProcessorCities processorCities(m_temporaryMwmPath, affiliation, citiesHelper); + processorCities.SetPromoCatalog(m_citiesFilename); + processorCities.Process(); +} +} // namespace generator diff --git a/generator/final_processor_world.hpp b/generator/final_processor_world.hpp new file mode 100644 index 0000000000..8c838ad17f --- /dev/null +++ b/generator/final_processor_world.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "generator/feature_generator.hpp" +#include "generator/final_processor_interface.hpp" +#include "generator/world_map_generator.hpp" + +#include + +namespace generator +{ +class WorldFinalProcessor : public FinalProcessorIntermediateMwmInterface +{ +public: + using WorldGenerator = WorldMapGenerator; + + explicit WorldFinalProcessor(std::string const & temporaryMwmPath, + std::string const & coastlineGeomFilename); + + void SetPopularPlaces(std::string const & filename); + void SetCitiesAreas(std::string const & filename); + void SetPromoCatalog(std::string const & filename); + + // FinalProcessorIntermediateMwmInterface overrides: + void Process() override; + +private: + void ProcessCities(); + + std::string m_temporaryMwmPath; + std::string m_worldTmpFilename; + std::string m_coastlineGeomFilename; + std::string m_popularPlacesFilename; + std::string m_citiesAreasTmpFilename; + std::string m_citiesFilename; +}; +} // namespace generator diff --git a/generator/generator_integration_tests/features_tests.cpp b/generator/generator_integration_tests/features_tests.cpp index cd366c6545..6988d42178 100644 --- a/generator/generator_integration_tests/features_tests.cpp +++ b/generator/generator_integration_tests/features_tests.cpp @@ -7,6 +7,7 @@ #include "generator/generate_info.hpp" #include "generator/raw_generator.hpp" +#include "indexer/classificator.hpp" #include "indexer/classificator_loader.hpp" #include "indexer/ftypes_matcher.hpp" diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index f32d54e303..5730e14904 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -1,6 +1,10 @@ #include "generator/raw_generator.hpp" #include "generator/complex_loader.hpp" +#include "generator/features_processing_helpers.hpp" +#include "generator/final_processor_coastline.hpp" +#include "generator/final_processor_country.hpp" +#include "generator/final_processor_world.hpp" #include "generator/osm_source.hpp" #include "generator/processor_factory.hpp" #include "generator/raw_generator_writer.hpp" diff --git a/generator/raw_generator.hpp b/generator/raw_generator.hpp index f81e1bd2bf..b4cccf4544 100644 --- a/generator/raw_generator.hpp +++ b/generator/raw_generator.hpp @@ -1,7 +1,8 @@ #pragma once +#include "generator/composite_id.hpp" #include "generator/features_processing_helpers.hpp" -#include "generator/final_processor_intermediate_mwm.hpp" +#include "generator/final_processor_interface.hpp" #include "generator/generate_info.hpp" #include "generator/intermediate_data.hpp" #include "generator/translator_collection.hpp"