diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index 174bfb6ce8..ac77fa592e 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -88,8 +88,10 @@ set( filter_planet.hpp gen_mwm_info.hpp generate_info.hpp - geo_objects/geo_object_maintainer.cpp - geo_objects/geo_object_maintainer.hpp + geo_objects/geo_objects_generator.cpp + geo_objects/geo_objects_generator.hpp + geo_objects/geo_objects_maintainer.cpp + geo_objects/geo_objects_maintainer.hpp geo_objects/geo_objects.cpp geo_objects/geo_objects.hpp geo_objects/geo_objects_filter.cpp diff --git a/generator/generator_tests/geo_objects_tests.cpp b/generator/generator_tests/geo_objects_tests.cpp index f44a7bc630..360e7e65ba 100644 --- a/generator/generator_tests/geo_objects_tests.cpp +++ b/generator/generator_tests/geo_objects_tests.cpp @@ -5,9 +5,9 @@ #include "generator/feature_builder.hpp" #include "generator/feature_generator.hpp" #include "generator/generator_tests/common.hpp" -#include "generator/geo_objects/geo_object_maintainer.hpp" -#include "generator/geo_objects/geo_objects.hpp" #include "generator/geo_objects/geo_objects_filter.hpp" +#include "generator/geo_objects/geo_objects_generator.hpp" +#include "generator/geo_objects/geo_objects_maintainer.hpp" #include "indexer/classificator_loader.hpp" diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 6d60d34e4c..de3ba22704 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -12,7 +12,7 @@ #include "generator/feature_generator.hpp" #include "generator/feature_sorter.hpp" #include "generator/generate_info.hpp" -#include "generator/geo_objects/geo_objects.hpp" +#include "generator/geo_objects/geo_objects_generator.hpp" #include "generator/locality_sorter.hpp" #include "generator/maxspeeds_builder.hpp" #include "generator/metalines_builder.hpp" diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp index 571c7adc98..b49f044a2b 100644 --- a/generator/geo_objects/geo_objects.cpp +++ b/generator/geo_objects/geo_objects.cpp @@ -1,15 +1,15 @@ -#include "generator/geo_objects/geo_objects.hpp" - #include "generator/feature_builder.hpp" #include "generator/feature_generator.hpp" -#include "generator/geo_objects/geo_object_maintainer.hpp" -#include "generator/geo_objects/geo_objects_filter.hpp" #include "generator/key_value_storage.hpp" #include "generator/locality_sorter.hpp" + +#include "generator/geo_objects/geo_objects.hpp" +#include "generator/geo_objects/geo_objects_filter.hpp" +#include "generator/geo_objects/geo_objects_maintainer.hpp" + #include "generator/regions/region_base.hpp" #include "indexer/classificator.hpp" -#include "indexer/ftypes_matcher.hpp" #include "indexer/locality_index.hpp" #include "indexer/locality_index_builder.hpp" @@ -21,16 +21,15 @@ #include "base/geo_object_id.hpp" -#include -#include -#include -#include -#include - #include #include "3party/jansson/myjansson.hpp" +#include +#include +#include +#include + using namespace feature; namespace generator @@ -83,40 +82,6 @@ base::JSONPtr AddAddress(FeatureBuilder const & fb, KeyValue const & regionKeyVa return result; } -void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( - KeyValueStorage & geoObjectsKv, - GeoObjectsGenerator::RegionInfoGetterProxy const & regionInfoGetter, - std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount) -{ - std::mutex updateMutex; - auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) { - if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb)) - return; - - auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint()); - if (!regionKeyValue) - return; - - auto const id = fb.GetMostGenericOsmId().GetEncodedId(); - auto jsonValue = AddAddress(fb, *regionKeyValue); - - std::lock_guard lock(updateMutex); - geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)}); - }; - - ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer); - LOG(LINFO, ("Added", geoObjectsKv.Size(), "geo objects with addresses.")); -} - -struct NullBuildingsInfo -{ - std::unordered_map m_addressPoints2Buildings; - // Quite possible to have many points for one building. We want to use - // their addresses for POIs according to buildings and have no idea how to distinguish between - // them, so take one random - std::unordered_map m_Buildings2AddressPoint; -}; - NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, size_t threadsCount) @@ -128,8 +93,8 @@ NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectM if (!GeoObjectsFilter::HasHouse(fb) || !fb.IsPoint()) return; - - // Можно искать не нуллбилдинги в кв, а те айдишгики, которых нет в кв, которое построено без нуллбилдингов. + // Можно искать не нуллбилдинги в кв, а те айдишгики, которых нет в кв, которое построено без + // нуллбилдингов. auto const buildingId = geoObjectMaintainer.SearchIdOfFirstMatchedObject( fb.GetKeyPoint(), [](JsonValue const & json) { return !JsonHasBuilding(json); }); @@ -234,36 +199,38 @@ size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjects return pointsEnriched; } -NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( - GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, - size_t threadsCount) +std::shared_ptr FindHouse(FeatureBuilder const & fb, + GeoObjectMaintainer const & geoObjectMaintainer, + NullBuildingsInfo const & buildingsInfo) { - auto const buildingInfo = - GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount); + std::shared_ptr house = + geoObjectMaintainer.FindFirstMatchedObject(fb.GetKeyPoint(), JsonHasBuilding); + if (house) + return house; - LOG(LINFO, ("Found", buildingInfo.m_addressPoints2Buildings.size(), - "address points with outer building geometry")); - LOG(LINFO, - ("Found", buildingInfo.m_Buildings2AddressPoint.size(), "helpful addressless buildings")); - auto const buildingGeometries = - GetBuildingsGeometry(pathInGeoObjectsTmpMwm, buildingInfo, threadsCount); - LOG(LINFO, ("Saved", buildingGeometries.size(), "buildings geometries")); + std::vector potentialIds = + geoObjectMaintainer.SearchObjectsInIndex(fb.GetKeyPoint()); - size_t const pointsCount = AddBuildingGeometriesToAddressPoints( - pathInGeoObjectsTmpMwm, buildingInfo, buildingGeometries, threadsCount); + for (base::GeoObjectId id : potentialIds) + { + auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id); + if (it != buildingsInfo.m_Buildings2AddressPoint.end()) + return geoObjectMaintainer.GetKeyValueStorage().Find(it->second.GetEncodedId()); + } - LOG(LINFO, (pointsCount, "address points were enriched with outer building geomery")); - return buildingInfo; + return {}; } -template -auto Measure(std::string activity, Activist && activist) +base::JSONPtr MakeJsonValueWithNameFromFeature(FeatureBuilder const & fb, JsonValue const & json) { - LOG(LINFO, ("Start", activity)); - auto timer = base::Timer(); - SCOPE_GUARD(_, [&]() { LOG(LINFO, ("Finish", activity, timer.ElapsedSeconds(), "seconds.")); }); + auto jsonWithAddress = json.MakeDeepCopyJson(); - return activist(); + auto properties = json_object_get(jsonWithAddress.get(), "properties"); + Localizator localizator(*properties); + localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName())); + + UpdateCoordinates(fb.GetKeyPoint(), jsonWithAddress); + return jsonWithAddress; } } // namespace @@ -298,38 +265,50 @@ boost::optional> MakeTempGeoObjectsIndex( return indexer::ReadIndex, MmapReader>(indexFile); } -std::shared_ptr FindHousePoi(FeatureBuilder const & fb, - GeoObjectMaintainer const & geoObjectMaintainer, - NullBuildingsInfo const & buildingsInfo) +void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( + KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter, + std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount) { - std::shared_ptr house = - geoObjectMaintainer.FindFirstMatchedObject(fb.GetKeyPoint(), JsonHasBuilding); - if (house) - return house; + std::mutex updateMutex; + auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) { + if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb)) + return; - std::vector potentialIds = - geoObjectMaintainer.SearchObjectsInIndex(fb.GetKeyPoint()); + auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint()); + if (!regionKeyValue) + return; - for (base::GeoObjectId id : potentialIds) - { - auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id); -// if (it != buildingsInfo.m_Buildings2AddressPoint.end()) -// return geoObjectMaintainer.GetKeyValueStorage().Find(it->second.GetEncodedId()); - } + auto const id = fb.GetMostGenericOsmId().GetEncodedId(); + auto jsonValue = AddAddress(fb, *regionKeyValue); - return {}; + std::lock_guard lock(updateMutex); + geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)}); + }; + + ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer); + LOG(LINFO, ("Added", geoObjectsKv.Size(), "geo objects with addresses.")); } -base::JSONPtr MakeJsonValueWithNameFromFeature(FeatureBuilder const & fb, JsonValue const & json) +NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( + GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, + size_t threadsCount) { - auto jsonWithAddress = json.MakeDeepCopyJson(); + auto const buildingInfo = + GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount); - auto properties = json_object_get(jsonWithAddress.get(), "properties"); - Localizator localizator(*properties); - localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName())); + LOG(LINFO, ("Found", buildingInfo.m_addressPoints2Buildings.size(), + "address points with outer building geometry")); + LOG(LINFO, + ("Found", buildingInfo.m_Buildings2AddressPoint.size(), "helpful addressless buildings")); + auto const buildingGeometries = + GetBuildingsGeometry(pathInGeoObjectsTmpMwm, buildingInfo, threadsCount); + LOG(LINFO, ("Saved", buildingGeometries.size(), "buildings geometries")); - UpdateCoordinates(fb.GetKeyPoint(), jsonWithAddress); - return jsonWithAddress; + size_t const pointsCount = AddBuildingGeometriesToAddressPoints( + pathInGeoObjectsTmpMwm, buildingInfo, buildingGeometries, threadsCount); + + LOG(LINFO, (pointsCount, "address points were enriched with outer building geomery")); + return buildingInfo; } void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv, @@ -348,7 +327,7 @@ void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv, if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb)) return; - auto const house = FindHousePoi(fb, geoObjectMaintainer, buildingsInfo); + auto const house = FindHouse(fb, geoObjectMaintainer, buildingsInfo); if (!house) return; @@ -388,83 +367,5 @@ void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pat ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentCollect); CHECK(base::RenameFileX(path, pathInGeoObjectsTmpMwm), ()); } - -GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex, - std::string pathInRegionsKv, - std::string pathInGeoObjectsTmpMwm, - std::string pathOutIdsWithoutAddress, - std::string pathOutGeoObjectsKv, bool verbose, - size_t threadsCount) - : m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm)) - , m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress)) - , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) - , m_verbose(verbose) - , m_threadsCount(threadsCount) - , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) - , m_regionInfoGetter(pathInRegionsIndex, pathInRegionsKv) - -{ -} - -GeoObjectsGenerator::GeoObjectsGenerator(RegionInfoGetter && regionInfoGetter, - std::string pathInGeoObjectsTmpMwm, - std::string pathOutIdsWithoutAddress, - std::string pathOutGeoObjectsKv, bool verbose, - size_t threadsCount) - : m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm)) - , m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress)) - , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) - , m_verbose(verbose) - , m_threadsCount(threadsCount) - , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) - , m_regionInfoGetter(std::move(regionInfoGetter)) -{ -} - -bool GeoObjectsGenerator::GenerateGeoObjects() -{ - return Measure("generating geo objects", [&]() { return GenerateGeoObjectsPrivate(); }); -} - -bool GeoObjectsGenerator::GenerateGeoObjectsPrivate() -{ - auto geoObjectIndexFuture = - std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm); - - AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( - m_geoObjectsKv, m_regionInfoGetter, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount); - - LOG(LINFO, ("Geo objects with addresses were built.")); - - auto geoObjectIndex = geoObjectIndexFuture.get(); - - LOG(LINFO, ("Index was built.")); - - if (!geoObjectIndex) - return false; - - GeoObjectMaintainer const geoObjectMaintainer{std::move(*geoObjectIndex), m_geoObjectsKv}; - - LOG(LINFO, ("Enrich address points with outer null building geometry.")); - - NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry( - geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount); - - std::ofstream streamPoiIdsToAddToLocalityIndex(m_pathOutPoiIdsToAddToLocalityIndex); - - AddPoisEnrichedWithHouseAddresses(m_geoObjectsKv, geoObjectMaintainer, buildingInfo, - m_pathInGeoObjectsTmpMwm, streamPoiIdsToAddToLocalityIndex, - m_verbose, m_threadsCount); - - FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo, - m_threadsCount); - - LOG(LINFO, ("Addressless buildings with geometry we used for inner points were filtered")); - - LOG(LINFO, ("Geo objects without addresses were built.")); - LOG(LINFO, ("Geo objects key-value storage saved to", m_pathOutGeoObjectsKv)); - LOG(LINFO, ("Ids of POIs without addresses saved to", m_pathOutPoiIdsToAddToLocalityIndex)); - return true; -} } // namespace geo_objects } // namespace generator diff --git a/generator/geo_objects/geo_objects.hpp b/generator/geo_objects/geo_objects.hpp index 671b9d1f3b..9086358a2c 100644 --- a/generator/geo_objects/geo_objects.hpp +++ b/generator/geo_objects/geo_objects.hpp @@ -1,8 +1,11 @@ #pragma once #include "generator/key_value_storage.hpp" + #include "generator/regions/region_info_getter.hpp" +#include "generator/geo_objects/geo_objects_maintainer.hpp" + #include "geometry/meter.hpp" #include "geometry/point2d.hpp" @@ -18,6 +21,34 @@ namespace generator { namespace geo_objects { +class RegionInfoGetterProxy +{ +public: + using RegionInfoGetter = std::function(m2::PointD const & pathPoint)>; + RegionInfoGetterProxy(std::string const & pathInRegionsIndex, + std::string const & pathInRegionsKv) + { + m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv); + LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size())); + } + + explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter) + : m_externalInfoGetter(std::move(regionInfoGetter)) + { + LOG(LINFO, ("External regions info provided")); + } + + boost::optional FindDeepest(m2::PointD const & point) const + { + return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point) + : m_externalInfoGetter->operator()(point); + } +private: + boost::optional m_regionInfoGetter; + boost::optional m_externalInfoGetter; +}; + + using IndexReader = ReaderPtr; boost::optional> MakeTempGeoObjectsIndex( @@ -25,74 +56,35 @@ boost::optional> MakeTempGeoObjectsIndex( bool JsonHasBuilding(JsonValue const & json); -class GeoObjectsGenerator +void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( + KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter, + std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount); + +struct NullBuildingsInfo { -public: - using RegionInfoGetter = std::function(m2::PointD const & pathPoint)>; - - class RegionInfoGetterProxy - { - public: - RegionInfoGetterProxy(std::string const & pathInRegionsIndex, - std::string const & pathInRegionsKv) - { - m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv); - LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size())); - } - - explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter) - : m_externalInfoGetter(std::move(regionInfoGetter)) - { - LOG(LINFO, ("External regions info provided")); - } - - boost::optional FindDeepest(m2::PointD const & point) const - { - return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point) - : m_externalInfoGetter->operator()(point); - } - private: - boost::optional m_regionInfoGetter; - boost::optional m_externalInfoGetter; - }; - - GeoObjectsGenerator(std::string pathInRegionsIndex, std::string pathInRegionsKv, - std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress, - std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount); - - GeoObjectsGenerator(RegionInfoGetter && regionInfoGetter, std::string pathInGeoObjectsTmpMwm, - std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv, - bool verbose, size_t threadsCount); - - // This function generates key-value pairs for geo objects. - // First, we try to generate key-value pairs only for houses, since we cannot say anything about - // poi. In this step, we need key-value pairs for the regions and the index for the regions. Then - // we build an index for houses. And then we finish building key-value pairs for poi using this - // index for houses. - bool GenerateGeoObjects(); - - KeyValueStorage const & GetKeyValueStorage() const - { - return m_geoObjectsKv; - } - -private: - bool GenerateGeoObjectsPrivate(); - static KeyValueStorage InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv) - { - Platform().RemoveFileIfExists(pathOutGeoObjectsKv); - return KeyValueStorage(pathOutGeoObjectsKv, 0 /* cacheValuesCountLimit */); - } - - std::string m_pathInGeoObjectsTmpMwm; - std::string m_pathOutPoiIdsToAddToLocalityIndex; - std::string m_pathOutGeoObjectsKv; - - bool m_verbose = false; - size_t m_threadsCount = 1; - - KeyValueStorage m_geoObjectsKv; - RegionInfoGetterProxy m_regionInfoGetter; + std::unordered_map m_addressPoints2Buildings; + // Quite possible to have many points for one building. We want to use + // their addresses for POIs according to buildings and have no idea how to distinguish between + // them, so take one random + std::unordered_map m_Buildings2AddressPoint; }; + +NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( + GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, + size_t threadsCount); + + +void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv, + GeoObjectMaintainer const & geoObjectMaintainer, + NullBuildingsInfo const & buildingsInfo, + std::string const & pathInGeoObjectsTmpMwm, + std::ostream & streamPoiIdsToAddToLocalityIndex, + bool verbose, size_t threadsCount); + + +void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pathInGeoObjectsTmpMwm, + NullBuildingsInfo const & buildingsInfo, + size_t threadsCount); + } // namespace geo_objects } // namespace generator diff --git a/generator/geo_objects/geo_objects_generator.cpp b/generator/geo_objects/geo_objects_generator.cpp new file mode 100644 index 0000000000..19530cf480 --- /dev/null +++ b/generator/geo_objects/geo_objects_generator.cpp @@ -0,0 +1,104 @@ +#include "generator/geo_objects/geo_objects_generator.hpp" + +#include "base/logging.hpp" +#include "base/scope_guard.hpp" +#include "base/timer.hpp" + +#include + +namespace +{ +template +auto Measure(std::string activity, Activist && activist) +{ + LOG(LINFO, ("Start", activity)); + auto timer = base::Timer(); + SCOPE_GUARD(_, [&]() { LOG(LINFO, ("Finish", activity, timer.ElapsedSeconds(), "seconds.")); }); + + return activist(); +} +} // namespace + +namespace generator +{ +namespace geo_objects +{ +GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex, + std::string pathInRegionsKv, + std::string pathInGeoObjectsTmpMwm, + std::string pathOutIdsWithoutAddress, + std::string pathOutGeoObjectsKv, bool verbose, + size_t threadsCount) + : m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm)) + , m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress)) + , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) + , m_verbose(verbose) + , m_threadsCount(threadsCount) + , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) + , m_regionInfoGetter(pathInRegionsIndex, pathInRegionsKv) + +{ +} + +GeoObjectsGenerator::GeoObjectsGenerator( + RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter, std::string pathInGeoObjectsTmpMwm, + std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv, bool verbose, + size_t threadsCount) + : m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm)) + , m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress)) + , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) + , m_verbose(verbose) + , m_threadsCount(threadsCount) + , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) + , m_regionInfoGetter(std::move(regionInfoGetter)) +{ +} + +bool GeoObjectsGenerator::GenerateGeoObjects() +{ + return Measure("generating geo objects", [&]() { return GenerateGeoObjectsPrivate(); }); +} + +bool GeoObjectsGenerator::GenerateGeoObjectsPrivate() +{ + auto geoObjectIndexFuture = + std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm); + + AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( + m_geoObjectsKv, m_regionInfoGetter, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount); + + LOG(LINFO, ("Geo objects with addresses were built.")); + + auto geoObjectIndex = geoObjectIndexFuture.get(); + + LOG(LINFO, ("Index was built.")); + + if (!geoObjectIndex) + return false; + + GeoObjectMaintainer const geoObjectMaintainer{std::move(*geoObjectIndex), m_geoObjectsKv}; + + LOG(LINFO, ("Enrich address points with outer null building geometry.")); + + NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry( + geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount); + + std::ofstream streamPoiIdsToAddToLocalityIndex(m_pathOutPoiIdsToAddToLocalityIndex); + + AddPoisEnrichedWithHouseAddresses(m_geoObjectsKv, geoObjectMaintainer, buildingInfo, + m_pathInGeoObjectsTmpMwm, streamPoiIdsToAddToLocalityIndex, + m_verbose, m_threadsCount); + + FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo, + m_threadsCount); + + LOG(LINFO, ("Addressless buildings with geometry we used for inner points were filtered")); + + LOG(LINFO, ("Geo objects without addresses were built.")); + LOG(LINFO, ("Geo objects key-value storage saved to", m_pathOutGeoObjectsKv)); + LOG(LINFO, ("Ids of POIs without addresses saved to", m_pathOutPoiIdsToAddToLocalityIndex)); + return true; +} + +} // namespace geo_objects +} // namespace generator diff --git a/generator/geo_objects/geo_objects_generator.hpp b/generator/geo_objects/geo_objects_generator.hpp new file mode 100644 index 0000000000..a7e1120185 --- /dev/null +++ b/generator/geo_objects/geo_objects_generator.hpp @@ -0,0 +1,60 @@ +#include "generator/key_value_storage.hpp" + +#include "generator/regions/region_info_getter.hpp" + +#include "generator/geo_objects/geo_objects.hpp" + +#include "geometry/meter.hpp" +#include "geometry/point2d.hpp" + +#include "platform/platform.hpp" + +#include + +namespace generator +{ +namespace geo_objects +{ +class GeoObjectsGenerator +{ +public: + GeoObjectsGenerator(std::string pathInRegionsIndex, std::string pathInRegionsKv, + std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress, + std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount); + + GeoObjectsGenerator(RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter, + std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress, + std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount); + + // This function generates key-value pairs for geo objects. + // First, we try to generate key-value pairs only for houses, since we cannot say anything about + // poi. In this step, we need key-value pairs for the regions and the index for the regions. Then + // we build an index for houses. And then we finish building key-value pairs for poi using this + // index for houses. + bool GenerateGeoObjects(); + + KeyValueStorage const & GetKeyValueStorage() const + { + return m_geoObjectsKv; + } + +private: + bool GenerateGeoObjectsPrivate(); + static KeyValueStorage InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv) + { + Platform().RemoveFileIfExists(pathOutGeoObjectsKv); + return KeyValueStorage(pathOutGeoObjectsKv, 0 /* cacheValuesCountLimit */); + } + + std::string m_pathInGeoObjectsTmpMwm; + std::string m_pathOutPoiIdsToAddToLocalityIndex; + std::string m_pathOutGeoObjectsKv; + + bool m_verbose = false; + size_t m_threadsCount = 1; + + KeyValueStorage m_geoObjectsKv; + RegionInfoGetterProxy m_regionInfoGetter; +}; +} // namespace geo_objects +} // namespace generator diff --git a/generator/geo_objects/geo_object_maintainer.cpp b/generator/geo_objects/geo_objects_maintainer.cpp similarity index 96% rename from generator/geo_objects/geo_object_maintainer.cpp rename to generator/geo_objects/geo_objects_maintainer.cpp index 67cb6476ec..c4b8deef56 100644 --- a/generator/geo_objects/geo_object_maintainer.cpp +++ b/generator/geo_objects/geo_objects_maintainer.cpp @@ -1,4 +1,4 @@ -#include "generator/geo_objects/geo_object_maintainer.hpp" +#include "generator/geo_objects/geo_objects_maintainer.hpp" #include diff --git a/generator/geo_objects/geo_object_maintainer.hpp b/generator/geo_objects/geo_objects_maintainer.hpp similarity index 94% rename from generator/geo_objects/geo_object_maintainer.hpp rename to generator/geo_objects/geo_objects_maintainer.hpp index 64441da825..28a4d38ceb 100644 --- a/generator/geo_objects/geo_object_maintainer.hpp +++ b/generator/geo_objects/geo_objects_maintainer.hpp @@ -42,6 +42,10 @@ public: return SearchGeoObjectIdsByPoint(m_index, point); } + KeyValueStorage const & GetKeyValueStorage() const + { + return m_storage; + } static std::vector SearchGeoObjectIdsByPoint( indexer::GeoObjectsIndex const & index, m2::PointD const & point);