diff --git a/generator/generator_tests/geo_objects_tests.cpp b/generator/generator_tests/geo_objects_tests.cpp index 360e7e65ba..d354dff1f7 100644 --- a/generator/generator_tests/geo_objects_tests.cpp +++ b/generator/generator_tests/geo_objects_tests.cpp @@ -25,7 +25,7 @@ bool CheckWeGotExpectedIdsByPoint(m2::PointD const & point, indexer::GeoObjectsIndex const & index) { std::vector test = - GeoObjectMaintainer::SearchGeoObjectIdsByPoint(index, point); + GeoObjectMaintainer::GeoObjectsView::SearchGeoObjectIdsByPoint(index, point); std::sort(test.begin(), test.end()); std::sort(reference.begin(), reference.end()); return test == reference; @@ -52,10 +52,10 @@ std::vector CollectFeatures( return expectedIds; } -GeoObjectsGenerator TearUp(std::vector const & osmElements, - ScopedFile const & geoObjectsFeatures, - ScopedFile const & idsWithoutAddresses, - ScopedFile const & geoObjectsKeyValue) +std::unique_ptr TearUp(std::vector const & osmElements, + ScopedFile const & geoObjectsFeatures, + ScopedFile const & idsWithoutAddresses, + ScopedFile const & geoObjectsKeyValue) { // Absolutely random region. std::shared_ptr value = std::make_shared(LoadFromString( @@ -100,16 +100,24 @@ GeoObjectsGenerator TearUp(std::vector const & osmElements, auto regionGetter = [value](auto && point) { return KeyValue{1, value}; }; - return GeoObjectsGenerator{regionGetter, - geoObjectsFeatures.GetFullPath(), - idsWithoutAddresses.GetFullPath(), - geoObjectsKeyValue.GetFullPath(), - false /* verbose */, - 1 /* threadsCount */}; + return std::make_unique( + regionGetter, geoObjectsFeatures.GetFullPath(), idsWithoutAddresses.GetFullPath(), + geoObjectsKeyValue.GetFullPath(), false /* verbose */, 1 /* threadsCount */); +} + +void TestRegionAddress(json_t const * json) +{ + const auto address = + GetJSONObligatoryFieldByPath(json, "properties", "locales", "default", "address"); + TEST_EQUAL(FromJSONToString(GetJSONObligatoryField(address, "country")), "Bahamas", ()); + TEST_EQUAL(FromJSONToString(GetJSONObligatoryField(address, "region")), "Central Abaco", ()); + TEST_EQUAL(FromJSONToString(GetJSONObligatoryField(address, "locality")), "Leisure Lee", ()); } void TestFindReverse(std::vector const & osmElements, - std::vector const & where) + std::vector const & where, + std::vector const & expected = {} + ) { classificator::Load(); ScopedFile const geoObjectsFeatures{"geo_objects_features.mwm", ScopedFile::Mode::DoNotCreate}; @@ -119,10 +127,10 @@ void TestFindReverse(std::vector const & osmElements, auto const & expectedIds = CollectFeatures( osmElements, geoObjectsFeatures, [](FeatureBuilder const & fb) { return fb.IsPoint(); }); - GeoObjectsGenerator geoObjectsGenerator = - TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue); + std::unique_ptr geoObjectsGenerator{ + TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)}; - TEST(geoObjectsGenerator.GenerateGeoObjects(), ("Generate Geo Objects failed")); + TEST(geoObjectsGenerator->GenerateGeoObjects(), ("Generate Geo Objects failed")); auto const geoObjectsIndex = MakeTempGeoObjectsIndex(geoObjectsFeatures.GetFullPath()); @@ -132,6 +140,30 @@ void TestFindReverse(std::vector const & osmElements, { TEST(CheckWeGotExpectedIdsByPoint(point, expectedIds, *geoObjectsIndex), ()); } + + auto const & view = geoObjectsGenerator->GetMaintainer().CreateView(); + + const auto & toCheck = expected.empty() ? expectedIds : expected; + std::cerr << DebugPrint(toCheck) << std::endl; + + for (auto const & id : toCheck) + { + auto json = view.GetFullGeoObject(id); + TestRegionAddress(json.get()); + TEST(JsonHasBuilding(JsonValue{std::move(json)}), ("No address for", id)); + } + + geoObjectsGenerator.reset(nullptr); + + KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath(), 0 /*cacheValuesCountLimit*/}; + + for (GeoObjectId id : toCheck) + { + std::shared_ptr value = kvStorage.Find(id.GetEncodedId()); + TEST(value, ("Id", id.GetEncodedId(), "is not stored in key value")); + TEST(JsonHasBuilding(*value), ("No address for", id)); + TestRegionAddress(*value); + } } UNIT_TEST(GenerateGeoObjects_AddNullBuildingGeometryForPointsWithAddressesInside) @@ -148,24 +180,20 @@ UNIT_TEST(GenerateGeoObjects_AddNullBuildingGeometryForPointsWithAddressesInside {{1, 1}, {4, 4}}, {}}, {3, - {{"addr:housenumber", "39 с80"}, - {"addr:street", "Ленинградский проспект"}}, + {{"addr:housenumber", "39 с80"}, {"addr:street", "Ленинградский проспект"}}, {{1.6, 1.6}}, - {}} - }; + {}}}; TestFindReverse(osmElements, {{1.5, 1.5}, {2, 2}, {4, 4}}); } -UNIT_TEST(GenerateGeoObjects_AddNullBuildingGeometryForPointsWithAddressesInside2) +UNIT_TEST(GenerateGeoObjects_AddNullBuildingGeometryForPointsWithAddressesReverse) { std::vector const osmElements{ {1, - {{"addr:housenumber", "39 с80"}, - {"addr:street", "Ленинградский проспект"}}, + {{"addr:housenumber", "39 с80"}, {"addr:street", "Ленинградский проспект"}}, {{1.6, 1.6}}, - {} - }, + {}}, {2, {{"addr:housenumber", "39 с79"}, {"addr:street", "Ленинградский проспект"}, @@ -195,7 +223,7 @@ UNIT_TEST(GenerateGeoObjects_AddNullBuildingPointToPoint) {{1.5, 1.5}}, {}}, }; - TestFindReverse(osmElements, {}); + TestFindReverse(osmElements, {}, {base::MakeOsmNode(1)}); } void TestPoiHasAddress(std::vector const & osmElements) @@ -209,20 +237,25 @@ void TestPoiHasAddress(std::vector const & osmElements) CollectFeatures(osmElements, geoObjectsFeatures, [](FeatureBuilder const & fb) { return GeoObjectsFilter::IsPoi(fb); }); - GeoObjectsGenerator geoObjectsGenerator = - TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue); + std::unique_ptr geoObjectsGenerator = { + TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)}; + + TEST(geoObjectsGenerator->GenerateGeoObjects(), ("Generate Geo Objects failed")); + + geoObjectsGenerator.reset(nullptr); + + KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath(), 0 /*cacheValuesCountLimit*/}; - TEST(geoObjectsGenerator.GenerateGeoObjects(), ("Generate Geo Objects failed")); for (GeoObjectId id : expectedIds) { - std::shared_ptr value = - geoObjectsGenerator.GetKeyValueStorage().Find(id.GetEncodedId()); + std::shared_ptr value = kvStorage.Find(id.GetEncodedId()); TEST(value, ("Id", id, "is not stored in key value")); TEST(JsonHasBuilding(*value), ("No address for", id)); + TestRegionAddress(*value); } } -UNIT_TEST(GenerateGeoObjects_CheckPoiEnricedWithAddress) +UNIT_TEST(GenerateGeoObjects_CheckPoiEnrichedWithAddress) { std::vector const osmElements{ {1, {{"addr:housenumber", "111"}, {"addr:street", "Healing street"}}, {{1.6, 1.6}}, {}}, diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp index b49f044a2b..b1aee33bca 100644 --- a/generator/geo_objects/geo_objects.cpp +++ b/generator/geo_objects/geo_objects.cpp @@ -38,66 +38,27 @@ namespace geo_objects { namespace { -void UpdateCoordinates(m2::PointD const & point, base::JSONPtr & json) -{ - auto geometry = json_object_get(json.get(), "geometry"); - auto coordinates = json_object_get(geometry, "coordinates"); - if (json_array_size(coordinates) == 2) - { - auto const latLon = MercatorBounds::ToLatLon(point); - json_array_set_new(coordinates, 0, ToJSON(latLon.m_lon).release()); - json_array_set_new(coordinates, 1, ToJSON(latLon.m_lat).release()); - } -} - -base::JSONPtr AddAddress(FeatureBuilder const & fb, KeyValue const & regionKeyValue) -{ - auto result = regionKeyValue.second->MakeDeepCopyJson(); - - UpdateCoordinates(fb.GetKeyPoint(), result); - - auto properties = base::GetJSONObligatoryField(result.get(), "properties"); - auto address = base::GetJSONObligatoryFieldByPath(properties, "locales", "default", "address"); - auto const street = fb.GetParams().GetStreet(); - if (!street.empty()) - ToJSONObject(*address, "street", street); - - // By writing home null in the field we can understand that the house has no address. - auto const house = fb.GetParams().house.Get(); - if (!house.empty()) - ToJSONObject(*address, "building", house); - else - ToJSONObject(*address, "building", base::NewJSONNull()); - - Localizator localizator(*properties); - localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName())); - - int const kHouseOrPoiRank = 30; - ToJSONObject(*properties, "rank", kHouseOrPoiRank); - - ToJSONObject(*properties, "dref", KeyValueStorage::SerializeDref(regionKeyValue.first)); - // auto locales = json_object_get(result.get(), "locales"); - // auto en = json_object_get(result.get(), "en"); - // todo(maksimandrianov): Add en locales. - return result; -} - -NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectMaintainer, +NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, size_t threadsCount) { NullBuildingsInfo result; static int64_t counter = 0; std::mutex updateMutex; + auto const & view = geoObjectMaintainer.CreateView(); + auto const saveIdFold = [&](FeatureBuilder & fb, uint64_t /* currPos */) { if (!GeoObjectsFilter::HasHouse(fb) || !fb.IsPoint()) return; - // Можно искать не нуллбилдинги в кв, а те айдишгики, которых нет в кв, которое построено без - // нуллбилдингов. + // search for ids of Buildinds not stored with geoObjectsMantainer + // they are nullBuildings + auto const buildingId = + view.SearchIdOfFirstMatchedObject(fb.GetKeyPoint(), [&view](base::GeoObjectId id) { + auto const & geoData = view.GetGeoData(id); + return geoData && geoData->m_house.empty(); + }); - auto const buildingId = geoObjectMaintainer.SearchIdOfFirstMatchedObject( - fb.GetKeyPoint(), [](JsonValue const & json) { return !JsonHasBuilding(json); }); if (!buildingId) return; @@ -106,7 +67,7 @@ NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectM std::lock_guard lock(updateMutex); result.m_addressPoints2Buildings[id] = *buildingId; counter++; - if (counter % 100000) + if (counter % 100000 == 0) LOG(LINFO, (counter, "Helpful building added")); result.m_Buildings2AddressPoint[*buildingId] = id; }; @@ -141,12 +102,12 @@ BuildingsGeometries GetBuildingsGeometry(std::string const & pathInGeoObjectsTmp result[id] = fb.GetGeometry(); counter++; - if (counter % 100000) + if (counter % 100000 == 0) LOG(LINFO, (counter, "Building geometries added")); }; ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, saveIdFold); - LOG(LINFO, (sizeof(result), " is size of geometries")); + LOG(LINFO, (sizeof(result), "is size of geometries in bytes")); return result; } @@ -181,7 +142,7 @@ size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjects fb.PreSerialize(); ++pointsEnriched; - if (pointsEnriched % 100000) + if (pointsEnriched % 100000 == 0) LOG(LINFO, (pointsEnriched, "Points enriched with geometry")); } else @@ -199,23 +160,26 @@ size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjects return pointsEnriched; } -std::shared_ptr FindHouse(FeatureBuilder const & fb, - GeoObjectMaintainer const & geoObjectMaintainer, - NullBuildingsInfo const & buildingsInfo) +base::JSONPtr FindHouse(FeatureBuilder const & fb, GeoObjectMaintainer & geoObjectMaintainer, + NullBuildingsInfo const & buildingsInfo) { - std::shared_ptr house = - geoObjectMaintainer.FindFirstMatchedObject(fb.GetKeyPoint(), JsonHasBuilding); + auto const & view = geoObjectMaintainer.CreateView(); + + base::JSONPtr house = + view.GetFullGeoObject(fb.GetKeyPoint(), [](GeoObjectMaintainer::GeoObjectData const & data) { + return !data.m_house.empty(); + }); + if (house) return house; - std::vector potentialIds = - geoObjectMaintainer.SearchObjectsInIndex(fb.GetKeyPoint()); + std::vector potentialIds = view.SearchObjectsInIndex(fb.GetKeyPoint()); for (base::GeoObjectId id : potentialIds) { auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id); if (it != buildingsInfo.m_Buildings2AddressPoint.end()) - return geoObjectMaintainer.GetKeyValueStorage().Find(it->second.GetEncodedId()); + return view.GetFullGeoObject(it->second); } return {}; @@ -266,32 +230,20 @@ boost::optional> MakeTempGeoObjectsIndex( } void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( - KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter, - std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount) + GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, + bool verbose, size_t threadsCount) { - std::mutex updateMutex; auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) { - if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb)) - return; - - auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint()); - if (!regionKeyValue) - return; - - auto const id = fb.GetMostGenericOsmId().GetEncodedId(); - auto jsonValue = AddAddress(fb, *regionKeyValue); - - std::lock_guard lock(updateMutex); - geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)}); + geoObjectMaintainer.StoreAndEnrich(fb); }; ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer); - LOG(LINFO, ("Added", geoObjectsKv.Size(), "geo objects with addresses.")); + LOG(LINFO, ("Added", geoObjectMaintainer.Size(), "geo objects with addresses.")); } -NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( - GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, - size_t threadsCount) +NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(GeoObjectMaintainer & geoObjectMaintainer, + std::string const & pathInGeoObjectsTmpMwm, + size_t threadsCount) { auto const buildingInfo = GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount); @@ -311,40 +263,37 @@ NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( return buildingInfo; } -void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv, - GeoObjectMaintainer const & geoObjectMaintainer, +void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer, NullBuildingsInfo const & buildingsInfo, std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamPoiIdsToAddToLocalityIndex, bool verbose, size_t threadsCount) { - auto const addressObjectsCount = geoObjectsKv.Size(); - size_t counter = 0; - std::mutex updateMutex; + std::atomic_size_t counter{0}; + auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) { if (!GeoObjectsFilter::IsPoi(fb)) return; if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb)) return; - auto const house = FindHouse(fb, geoObjectMaintainer, buildingsInfo); + auto house = FindHouse(fb, geoObjectMaintainer, buildingsInfo); if (!house) return; - auto const id = fb.GetMostGenericOsmId().GetEncodedId(); - auto jsonValue = MakeJsonValueWithNameFromFeature(fb, *house); + auto const id = fb.GetMostGenericOsmId(); + auto jsonValue = MakeJsonValueWithNameFromFeature(fb, JsonValue{std::move(house)}); - std::lock_guard lock(updateMutex); counter++; - if (counter % 100000) + if (counter % 100000 == 0) LOG(LINFO, (counter, "pois added added")); - geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)}); + + geoObjectMaintainer.WriteToStorage(id, JsonValue{std::move(jsonValue)}); streamPoiIdsToAddToLocalityIndex << id << "\n"; }; ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer); - LOG(LINFO, - ("Added ", geoObjectsKv.Size() - addressObjectsCount, "geo objects without addresses.")); + LOG(LINFO, ("Added", counter, "POIs enriched with address.")); } void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pathInGeoObjectsTmpMwm, diff --git a/generator/geo_objects/geo_objects.hpp b/generator/geo_objects/geo_objects.hpp index 9086358a2c..5fad0fc189 100644 --- a/generator/geo_objects/geo_objects.hpp +++ b/generator/geo_objects/geo_objects.hpp @@ -2,8 +2,6 @@ #include "generator/key_value_storage.hpp" -#include "generator/regions/region_info_getter.hpp" - #include "generator/geo_objects/geo_objects_maintainer.hpp" #include "geometry/meter.hpp" @@ -21,33 +19,6 @@ namespace generator { namespace geo_objects { -class RegionInfoGetterProxy -{ -public: - using RegionInfoGetter = std::function(m2::PointD const & pathPoint)>; - RegionInfoGetterProxy(std::string const & pathInRegionsIndex, - std::string const & pathInRegionsKv) - { - m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv); - LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size())); - } - - explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter) - : m_externalInfoGetter(std::move(regionInfoGetter)) - { - LOG(LINFO, ("External regions info provided")); - } - - boost::optional FindDeepest(m2::PointD const & point) const - { - return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point) - : m_externalInfoGetter->operator()(point); - } -private: - boost::optional m_regionInfoGetter; - boost::optional m_externalInfoGetter; -}; - using IndexReader = ReaderPtr; @@ -57,8 +28,8 @@ boost::optional> MakeTempGeoObjectsIndex( bool JsonHasBuilding(JsonValue const & json); void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( - KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter, - std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount); + GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, + bool verbose, size_t threadsCount); struct NullBuildingsInfo { @@ -70,12 +41,10 @@ struct NullBuildingsInfo }; NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry( - GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, + GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm, size_t threadsCount); - -void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv, - GeoObjectMaintainer const & geoObjectMaintainer, +void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer, NullBuildingsInfo const & buildingsInfo, std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamPoiIdsToAddToLocalityIndex, diff --git a/generator/geo_objects/geo_objects_generator.cpp b/generator/geo_objects/geo_objects_generator.cpp index 19530cf480..62be262b7c 100644 --- a/generator/geo_objects/geo_objects_generator.cpp +++ b/generator/geo_objects/geo_objects_generator.cpp @@ -34,8 +34,7 @@ GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex, , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) , m_verbose(verbose) , m_threadsCount(threadsCount) - , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) - , m_regionInfoGetter(pathInRegionsIndex, pathInRegionsKv) + , m_geoObjectMaintainer{m_pathOutGeoObjectsKv, RegionInfoGetterProxy(pathInRegionsIndex, pathInRegionsKv)} { } @@ -49,8 +48,7 @@ GeoObjectsGenerator::GeoObjectsGenerator( , m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv)) , m_verbose(verbose) , m_threadsCount(threadsCount) - , m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv)) - , m_regionInfoGetter(std::move(regionInfoGetter)) + , m_geoObjectMaintainer{m_pathOutGeoObjectsKv, RegionInfoGetterProxy(std::move(regionInfoGetter))} { } @@ -65,7 +63,7 @@ bool GeoObjectsGenerator::GenerateGeoObjectsPrivate() std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm); AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses( - m_geoObjectsKv, m_regionInfoGetter, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount); + m_geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount); LOG(LINFO, ("Geo objects with addresses were built.")); @@ -76,18 +74,17 @@ bool GeoObjectsGenerator::GenerateGeoObjectsPrivate() if (!geoObjectIndex) return false; - GeoObjectMaintainer const geoObjectMaintainer{std::move(*geoObjectIndex), m_geoObjectsKv}; + m_geoObjectMaintainer.SetIndex(std::move(*geoObjectIndex)); LOG(LINFO, ("Enrich address points with outer null building geometry.")); NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry( - geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount); + m_geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount); std::ofstream streamPoiIdsToAddToLocalityIndex(m_pathOutPoiIdsToAddToLocalityIndex); - AddPoisEnrichedWithHouseAddresses(m_geoObjectsKv, geoObjectMaintainer, buildingInfo, - m_pathInGeoObjectsTmpMwm, streamPoiIdsToAddToLocalityIndex, - m_verbose, m_threadsCount); + AddPoisEnrichedWithHouseAddresses(m_geoObjectMaintainer, buildingInfo, m_pathInGeoObjectsTmpMwm, + streamPoiIdsToAddToLocalityIndex, m_verbose, m_threadsCount); FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo, m_threadsCount); diff --git a/generator/geo_objects/geo_objects_generator.hpp b/generator/geo_objects/geo_objects_generator.hpp index a7e1120185..162a9be404 100644 --- a/generator/geo_objects/geo_objects_generator.hpp +++ b/generator/geo_objects/geo_objects_generator.hpp @@ -32,19 +32,13 @@ public: // we build an index for houses. And then we finish building key-value pairs for poi using this // index for houses. bool GenerateGeoObjects(); - - KeyValueStorage const & GetKeyValueStorage() const + GeoObjectMaintainer& GetMaintainer() { - return m_geoObjectsKv; + return m_geoObjectMaintainer; } private: bool GenerateGeoObjectsPrivate(); - static KeyValueStorage InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv) - { - Platform().RemoveFileIfExists(pathOutGeoObjectsKv); - return KeyValueStorage(pathOutGeoObjectsKv, 0 /* cacheValuesCountLimit */); - } std::string m_pathInGeoObjectsTmpMwm; std::string m_pathOutPoiIdsToAddToLocalityIndex; @@ -52,9 +46,7 @@ private: bool m_verbose = false; size_t m_threadsCount = 1; - - KeyValueStorage m_geoObjectsKv; - RegionInfoGetterProxy m_regionInfoGetter; + GeoObjectMaintainer m_geoObjectMaintainer; }; } // namespace geo_objects } // namespace generator diff --git a/generator/geo_objects/geo_objects_maintainer.cpp b/generator/geo_objects/geo_objects_maintainer.cpp index c4b8deef56..ad66cfc2a3 100644 --- a/generator/geo_objects/geo_objects_maintainer.cpp +++ b/generator/geo_objects/geo_objects_maintainer.cpp @@ -1,4 +1,7 @@ #include "generator/geo_objects/geo_objects_maintainer.hpp" +#include "generator/geo_objects/geo_objects_filter.hpp" + +#include "generator/translation.hpp" #include @@ -6,47 +9,170 @@ namespace generator { namespace geo_objects { -GeoObjectMaintainer::GeoObjectMaintainer(indexer::GeoObjectsIndex && index, - KeyValueStorage const & kvStorage) - : m_index{std::move(index)}, m_storage{kvStorage} -{ } - -std::shared_ptr GeoObjectMaintainer::FindFirstMatchedObject( - m2::PointD const & point, std::function && pred) const +GeoObjectMaintainer::GeoObjectMaintainer(std::string const & pathOutGeoObjectsKv, + RegionInfoGetterProxy && regionInfoGetter) + : m_geoObjectsKvStorage{InitGeoObjectsKv(pathOutGeoObjectsKv)} + , m_regionInfoGetter{std::move(regionInfoGetter)} { - auto const ids = SearchGeoObjectIdsByPoint(m_index, point); +} + +// static +std::fstream GeoObjectMaintainer::InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv) +{ + std::fstream result{pathOutGeoObjectsKv, + std::ios_base::in | std::ios_base::out | std::ios_base::app}; + if (!result) + MYTHROW(Reader::OpenException, ("Failed to open file", pathOutGeoObjectsKv)); + + return std::move(result); +} + +void UpdateCoordinates(m2::PointD const & point, base::JSONPtr & json) +{ + auto geometry = json_object_get(json.get(), "geometry"); + auto coordinates = json_object_get(geometry, "coordinates"); + if (json_array_size(coordinates) == 2) + { + auto const latLon = MercatorBounds::ToLatLon(point); + json_array_set_new(coordinates, 0, ToJSON(latLon.m_lon).release()); + json_array_set_new(coordinates, 1, ToJSON(latLon.m_lat).release()); + } +} + +base::JSONPtr AddAddress(std::string const & street, std::string const & house, m2::PointD point, + StringUtf8Multilang const & name, KeyValue const & regionKeyValue) +{ + auto result = regionKeyValue.second->MakeDeepCopyJson(); + + UpdateCoordinates(point, result); + + auto properties = base::GetJSONObligatoryField(result.get(), "properties"); + auto address = base::GetJSONObligatoryFieldByPath(properties, "locales", "default", "address"); + if (!street.empty()) + ToJSONObject(*address, "street", street); + + // By writing home null in the field we can understand that the house has no address. + if (!house.empty()) + ToJSONObject(*address, "building", house); + else + ToJSONObject(*address, "building", base::NewJSONNull()); + + Localizator localizator(*properties); + localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(name)); + + int const kHouseOrPoiRank = 30; + ToJSONObject(*properties, "rank", kHouseOrPoiRank); + + ToJSONObject(*properties, "dref", KeyValueStorage::SerializeDref(regionKeyValue.first)); + // auto locales = json_object_get(result.get(), "locales"); + // auto locales = json_object_get(result.get(), "locales"); + // auto en = json_object_get(result.get(), "en"); + // todo(maksimandrianov): Add en locales. + return result; +} + +void GeoObjectMaintainer::StoreAndEnrich(feature::FeatureBuilder & fb) +{ + if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb)) + return; + + auto regionKeyValue = m_regionInfoGetter.FindDeepest(fb.GetKeyPoint()); + if (!regionKeyValue) + return; + + auto const id = fb.GetMostGenericOsmId(); + auto jsonValue = AddAddress(fb.GetParams().GetStreet(), fb.GetParams().house.Get(), + fb.GetKeyPoint(), fb.GetMultilangName(), *regionKeyValue); + + std::lock_guard lock(m_updateMutex); + + auto const it = m_geoId2GeoData.emplace( + std::make_pair(id, GeoObjectData{fb.GetParams().GetStreet(), fb.GetParams().house.Get(), + fb.GetKeyPoint(), fb.GetMultilangName()})); + + // Duplicate ID's are possible + if (!it.second) + return; + + m_geoObjectsKvStorage << KeyValueStorage::SerializeFullLine(id.GetEncodedId(), + JsonValue{std::move(jsonValue)}); +} + +void GeoObjectMaintainer::WriteToStorage(base::GeoObjectId id, JsonValue && value) +{ + std::lock_guard lock(m_updateMutex); + m_geoObjectsKvStorage << KeyValueStorage::SerializeFullLine(id.GetEncodedId(), std::move(value)); +} + +// GeoObjectMaintainer::GeoObjectsView + +base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObject( + m2::PointD point, + std::function && pred) const +{ + auto const ids = SearchGeoObjectIdsByPoint(m_geoIndex, point); for (auto const & id : ids) { - auto const object = m_storage.Find(id.GetEncodedId()); - if (!object) + auto const it = m_geoId2GeoData.find(id); + if (it == m_geoId2GeoData.end()) continue; - if (pred(std::cref(*object))) - return object; + auto const geoData = it->second; + if (!pred(geoData)) + continue; + + auto regionKeyValue = m_regionInfoGetter.FindDeepest(geoData.m_keyPoint); + if (!regionKeyValue) + return {}; + + return AddAddress(geoData.m_street, geoData.m_house, geoData.m_keyPoint, geoData.m_name, + *regionKeyValue); } return {}; } -boost::optional GeoObjectMaintainer::SearchIdOfFirstMatchedObject( - m2::PointD const & point, std::function && pred) const +base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObject(base::GeoObjectId id) const { - auto const ids = SearchGeoObjectIdsByPoint(m_index, point); + auto const it = m_geoId2GeoData.find(id); + if (it == m_geoId2GeoData.end()) + return {}; + + auto const geoData = it->second; + auto regionKeyValue = m_regionInfoGetter.FindDeepest(geoData.m_keyPoint); + if (!regionKeyValue) + return {}; + + return AddAddress(geoData.m_street, geoData.m_house, geoData.m_keyPoint, geoData.m_name, + *regionKeyValue); +} + +boost::optional GeoObjectMaintainer::GeoObjectsView::GetGeoData( + base::GeoObjectId id) const +{ + auto const it = m_geoId2GeoData.find(id); + if (it == m_geoId2GeoData.end()) + return {}; + + return it->second; +} + +boost::optional +GeoObjectMaintainer::GeoObjectsView::SearchIdOfFirstMatchedObject( + m2::PointD const & point, std::function && pred) const +{ + auto const ids = SearchObjectsInIndex(point); for (auto const & id : ids) { - auto const object = m_storage.Find(id.GetEncodedId()); - if (!object) - continue; - - if (pred(std::cref(*object))) + if (pred(id)) return id; } return {}; } -std::vector GeoObjectMaintainer::SearchGeoObjectIdsByPoint( - indexer::GeoObjectsIndex const & index, m2::PointD const & point) +std::vector GeoObjectMaintainer::GeoObjectsView::SearchGeoObjectIdsByPoint( + GeoIndex const & index, m2::PointD point) { std::vector ids; auto const emplace = [&ids](base::GeoObjectId const & osmId) { ids.emplace_back(osmId); }; diff --git a/generator/geo_objects/geo_objects_maintainer.hpp b/generator/geo_objects/geo_objects_maintainer.hpp index 28a4d38ceb..07e595b03e 100644 --- a/generator/geo_objects/geo_objects_maintainer.hpp +++ b/generator/geo_objects/geo_objects_maintainer.hpp @@ -2,6 +2,10 @@ #include "generator/key_value_storage.hpp" +#include "generator/regions/region_info_getter.hpp" + +#include "generator/feature_builder.hpp" + #include "indexer/locality_index.hpp" #include "coding/reader.hpp" @@ -23,36 +27,111 @@ namespace generator { namespace geo_objects { +class RegionInfoGetterProxy +{ +public: + using RegionInfoGetter = std::function(m2::PointD const & pathPoint)>; + RegionInfoGetterProxy(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv) + { + m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv); + LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size())); + } + + explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter) + : m_externalInfoGetter(std::move(regionInfoGetter)) + { + LOG(LINFO, ("External regions info provided")); + } + + boost::optional FindDeepest(m2::PointD const & point) const + { + return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point) + : m_externalInfoGetter->operator()(point); + } + +private: + boost::optional m_regionInfoGetter; + boost::optional m_externalInfoGetter; +}; + +void UpdateCoordinates(m2::PointD const & point, base::JSONPtr & json); + class GeoObjectMaintainer { public: - using IndexReader = ReaderPtr; - - GeoObjectMaintainer(indexer::GeoObjectsIndex && index, - KeyValueStorage const & kvStorage); - - std::shared_ptr FindFirstMatchedObject( - m2::PointD const & point, std::function && pred) const; - - boost::optional SearchIdOfFirstMatchedObject( - m2::PointD const & point, std::function && pred) const; - - std::vector SearchObjectsInIndex(m2::PointD const & point) const + struct GeoObjectData { - return SearchGeoObjectIdsByPoint(m_index, point); - } + std::string m_street; + std::string m_house; + m2::PointD m_keyPoint; + StringUtf8Multilang m_name; + }; - KeyValueStorage const & GetKeyValueStorage() const + using GeoId2GeoData = std::unordered_map; + using GeoIndex = indexer::GeoObjectsIndex>; + + class GeoObjectsView { - return m_storage; - } - static std::vector SearchGeoObjectIdsByPoint( - indexer::GeoObjectsIndex const & index, m2::PointD const & point); + public: + GeoObjectsView(GeoIndex const & geoIndex, GeoId2GeoData const & geoId2GeoData, + RegionInfoGetterProxy const & regionInfoGetter, std::mutex & updateMutex) + : m_geoIndex(geoIndex) + , m_geoId2GeoData(geoId2GeoData) + , m_regionInfoGetter(regionInfoGetter) + , m_lock(updateMutex, std::defer_lock) + { + CHECK(m_lock.try_lock(), ("Cannot create GeoObjectView on locked mutex")); + } + boost::optional SearchIdOfFirstMatchedObject( + m2::PointD const & point, std::function && pred) const; + boost::optional GetGeoData(base::GeoObjectId id) const; + + std::vector SearchObjectsInIndex(m2::PointD const & point) const + { + return SearchGeoObjectIdsByPoint(m_geoIndex, point); + } + + base::JSONPtr GetFullGeoObject(base::GeoObjectId id) const; + + base::JSONPtr GetFullGeoObject( + m2::PointD point, + std::function && pred) const; + + static std::vector SearchGeoObjectIdsByPoint(GeoIndex const & index, + m2::PointD point); + + private: + GeoIndex const & m_geoIndex; + GeoId2GeoData const & m_geoId2GeoData; + RegionInfoGetterProxy const & m_regionInfoGetter; + std::unique_lock m_lock; + }; + + GeoObjectMaintainer(std::string const & pathOutGeoObjectsKv, + RegionInfoGetterProxy && regionInfoGetter); + + void SetIndex(GeoIndex && index) { m_index = std::move(index); } + + void StoreAndEnrich(feature::FeatureBuilder & fb); + void WriteToStorage(base::GeoObjectId id, JsonValue && value); + + size_t Size() const { return m_geoId2GeoData.size(); } + + const GeoObjectsView CreateView() + { + return GeoObjectsView(m_index, m_geoId2GeoData, m_regionInfoGetter, m_updateMutex); + } private: - indexer::GeoObjectsIndex m_index; - KeyValueStorage const & m_storage; + static std::fstream InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv); + + std::fstream m_geoObjectsKvStorage; + std::mutex m_updateMutex; + + GeoIndex m_index; + RegionInfoGetterProxy m_regionInfoGetter; + GeoId2GeoData m_geoId2GeoData; }; } // namespace geo_objects } // namespace generator