AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] move some code to separate files, renamings

Browse source
This commit is contained in:
LaGrunge 2019-08-08 20:37:52 +03:00 • committed by cc-engineering
parent 1413bec7e7
commit 08b1210b5e
9 changed files with 307 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
generator/CMakeLists.txt
View file

@ -88,8 +88,10 @@ set(
filter_planet.hpp
gen_mwm_info.hpp
generate_info.hpp
geo_objects/geo_object_maintainer.cpp
geo_objects/geo_object_maintainer.hpp
geo_objects/geo_objects_generator.cpp
geo_objects/geo_objects_generator.hpp
geo_objects/geo_objects_maintainer.cpp
geo_objects/geo_objects_maintainer.hpp
geo_objects/geo_objects.cpp
geo_objects/geo_objects.hpp
geo_objects/geo_objects_filter.cpp

4
generator/generator_tests/geo_objects_tests.cpp
View file

@ -5,9 +5,9 @@
#include "generator/feature_builder.hpp"
#include "generator/feature_generator.hpp"
#include "generator/generator_tests/common.hpp"
#include "generator/geo_objects/geo_object_maintainer.hpp"
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/geo_objects/geo_objects_filter.hpp"
#include "generator/geo_objects/geo_objects_generator.hpp"
#include "generator/geo_objects/geo_objects_maintainer.hpp"
#include "indexer/classificator_loader.hpp"

2
generator/generator_tool/generator_tool.cpp
View file

@ -12,7 +12,7 @@
#include "generator/feature_generator.hpp"
#include "generator/feature_sorter.hpp"
#include "generator/generate_info.hpp"
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/geo_objects/geo_objects_generator.hpp"
#include "generator/locality_sorter.hpp"
#include "generator/maxspeeds_builder.hpp"
#include "generator/metalines_builder.hpp"

243
generator/geo_objects/geo_objects.cpp
View file

@ -1,15 +1,15 @@
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/feature_builder.hpp"
#include "generator/feature_generator.hpp"
#include "generator/geo_objects/geo_object_maintainer.hpp"
#include "generator/geo_objects/geo_objects_filter.hpp"
#include "generator/key_value_storage.hpp"
#include "generator/locality_sorter.hpp"
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/geo_objects/geo_objects_filter.hpp"
#include "generator/geo_objects/geo_objects_maintainer.hpp"
#include "generator/regions/region_base.hpp"
#include "indexer/classificator.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "indexer/locality_index.hpp"
#include "indexer/locality_index_builder.hpp"
@ -21,16 +21,15 @@
#include "base/geo_object_id.hpp"
#include <cstdint>
#include <fstream>
#include <functional>
#include <future>
#include <mutex>
#include <boost/optional.hpp>
#include "3party/jansson/myjansson.hpp"
#include <cstdint>
#include <fstream>
#include <functional>
#include <mutex>
using namespace feature;
namespace generator
@ -83,40 +82,6 @@ base::JSONPtr AddAddress(FeatureBuilder const & fb, KeyValue const & regionKeyVa
return result;
}
void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
KeyValueStorage & geoObjectsKv,
GeoObjectsGenerator::RegionInfoGetterProxy const & regionInfoGetter,
std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount)
{
std::mutex updateMutex;
auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) {
if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb))
return;
auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint());
if (!regionKeyValue)
return;
auto const id = fb.GetMostGenericOsmId().GetEncodedId();
auto jsonValue = AddAddress(fb, *regionKeyValue);
std::lock_guard<std::mutex> lock(updateMutex);
geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)});
};
ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer);
LOG(LINFO, ("Added", geoObjectsKv.Size(), "geo objects with addresses."));
}
struct NullBuildingsInfo
{
std::unordered_map<base::GeoObjectId, base::GeoObjectId> m_addressPoints2Buildings;
// Quite possible to have many points for one building. We want to use
// their addresses for POIs according to buildings and have no idea how to distinguish between
// them, so take one random
std::unordered_map<base::GeoObjectId, base::GeoObjectId> m_Buildings2AddressPoint;
};
NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectMaintainer,
std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount)
@ -128,8 +93,8 @@ NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer const & geoObjectM
if (!GeoObjectsFilter::HasHouse(fb) || !fb.IsPoint())
return;
// Можно искать не нуллбилдинги в кв, а те айдишгики, которых нет в кв, которое построено без нуллбилдингов.
// Можно искать не нуллбилдинги в кв, а те айдишгики, которых нет в кв, которое построено без
// нуллбилдингов.
auto const buildingId = geoObjectMaintainer.SearchIdOfFirstMatchedObject(
fb.GetKeyPoint(), [](JsonValue const & json) { return !JsonHasBuilding(json); });
@ -234,36 +199,38 @@ size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjects
return pointsEnriched;
}
NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(
GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount)
std::shared_ptr<JsonValue> FindHouse(FeatureBuilder const & fb,
GeoObjectMaintainer const & geoObjectMaintainer,
NullBuildingsInfo const & buildingsInfo)
{
auto const buildingInfo =
GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount);
std::shared_ptr<JsonValue> house =
geoObjectMaintainer.FindFirstMatchedObject(fb.GetKeyPoint(), JsonHasBuilding);
if (house)
return house;
LOG(LINFO, ("Found", buildingInfo.m_addressPoints2Buildings.size(),
"address points with outer building geometry"));
LOG(LINFO,
("Found", buildingInfo.m_Buildings2AddressPoint.size(), "helpful addressless buildings"));
auto const buildingGeometries =
GetBuildingsGeometry(pathInGeoObjectsTmpMwm, buildingInfo, threadsCount);
LOG(LINFO, ("Saved", buildingGeometries.size(), "buildings geometries"));
std::vector<base::GeoObjectId> potentialIds =
geoObjectMaintainer.SearchObjectsInIndex(fb.GetKeyPoint());
size_t const pointsCount = AddBuildingGeometriesToAddressPoints(
pathInGeoObjectsTmpMwm, buildingInfo, buildingGeometries, threadsCount);
for (base::GeoObjectId id : potentialIds)
{
auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id);
if (it != buildingsInfo.m_Buildings2AddressPoint.end())
return geoObjectMaintainer.GetKeyValueStorage().Find(it->second.GetEncodedId());
}
LOG(LINFO, (pointsCount, "address points were enriched with outer building geomery"));
return buildingInfo;
return {};
}
template <class Activist>
auto Measure(std::string activity, Activist && activist)
base::JSONPtr MakeJsonValueWithNameFromFeature(FeatureBuilder const & fb, JsonValue const & json)
{
LOG(LINFO, ("Start", activity));
auto timer = base::Timer();
SCOPE_GUARD(_, [&]() { LOG(LINFO, ("Finish", activity, timer.ElapsedSeconds(), "seconds.")); });
auto jsonWithAddress = json.MakeDeepCopyJson();
return activist();
auto properties = json_object_get(jsonWithAddress.get(), "properties");
Localizator localizator(*properties);
localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName()));
UpdateCoordinates(fb.GetKeyPoint(), jsonWithAddress);
return jsonWithAddress;
}
} // namespace
@ -298,38 +265,50 @@ boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
return indexer::ReadIndex<indexer::GeoObjectsIndexBox<IndexReader>, MmapReader>(indexFile);
}
std::shared_ptr<JsonValue> FindHousePoi(FeatureBuilder const & fb,
GeoObjectMaintainer const & geoObjectMaintainer,
NullBuildingsInfo const & buildingsInfo)
void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter,
std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount)
{
std::shared_ptr<JsonValue> house =
geoObjectMaintainer.FindFirstMatchedObject(fb.GetKeyPoint(), JsonHasBuilding);
if (house)
return house;
std::mutex updateMutex;
auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) {
if (!GeoObjectsFilter::IsBuilding(fb) && !GeoObjectsFilter::HasHouse(fb))
return;
std::vector<base::GeoObjectId> potentialIds =
geoObjectMaintainer.SearchObjectsInIndex(fb.GetKeyPoint());
auto regionKeyValue = regionInfoGetter.FindDeepest(fb.GetKeyPoint());
if (!regionKeyValue)
return;
for (base::GeoObjectId id : potentialIds)
{
auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id);
// if (it != buildingsInfo.m_Buildings2AddressPoint.end())
// return geoObjectMaintainer.GetKeyValueStorage().Find(it->second.GetEncodedId());
}
auto const id = fb.GetMostGenericOsmId().GetEncodedId();
auto jsonValue = AddAddress(fb, *regionKeyValue);
return {};
std::lock_guard<std::mutex> lock(updateMutex);
geoObjectsKv.Insert(id, JsonValue{std::move(jsonValue)});
};
ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentTransformer);
LOG(LINFO, ("Added", geoObjectsKv.Size(), "geo objects with addresses."));
}
base::JSONPtr MakeJsonValueWithNameFromFeature(FeatureBuilder const & fb, JsonValue const & json)
NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(
GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount)
{
auto jsonWithAddress = json.MakeDeepCopyJson();
auto const buildingInfo =
GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount);
auto properties = json_object_get(jsonWithAddress.get(), "properties");
Localizator localizator(*properties);
localizator.SetLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName()));
LOG(LINFO, ("Found", buildingInfo.m_addressPoints2Buildings.size(),
"address points with outer building geometry"));
LOG(LINFO,
("Found", buildingInfo.m_Buildings2AddressPoint.size(), "helpful addressless buildings"));
auto const buildingGeometries =
GetBuildingsGeometry(pathInGeoObjectsTmpMwm, buildingInfo, threadsCount);
LOG(LINFO, ("Saved", buildingGeometries.size(), "buildings geometries"));
UpdateCoordinates(fb.GetKeyPoint(), jsonWithAddress);
return jsonWithAddress;
size_t const pointsCount = AddBuildingGeometriesToAddressPoints(
pathInGeoObjectsTmpMwm, buildingInfo, buildingGeometries, threadsCount);
LOG(LINFO, (pointsCount, "address points were enriched with outer building geomery"));
return buildingInfo;
}
void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv,
@ -348,7 +327,7 @@ void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv,
if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb))
return;
auto const house = FindHousePoi(fb, geoObjectMaintainer, buildingsInfo);
auto const house = FindHouse(fb, geoObjectMaintainer, buildingsInfo);
if (!house)
return;
@ -388,83 +367,5 @@ void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pat
ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, concurrentCollect);
CHECK(base::RenameFileX(path, pathInGeoObjectsTmpMwm), ());
}
GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex,
std::string pathInRegionsKv,
std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
, m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress))
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
, m_verbose(verbose)
, m_threadsCount(threadsCount)
, m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv))
, m_regionInfoGetter(pathInRegionsIndex, pathInRegionsKv)
{
}
GeoObjectsGenerator::GeoObjectsGenerator(RegionInfoGetter && regionInfoGetter,
std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
, m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress))
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
, m_verbose(verbose)
, m_threadsCount(threadsCount)
, m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv))
, m_regionInfoGetter(std::move(regionInfoGetter))
{
}
bool GeoObjectsGenerator::GenerateGeoObjects()
{
return Measure("generating geo objects", [&]() { return GenerateGeoObjectsPrivate(); });
}
bool GeoObjectsGenerator::GenerateGeoObjectsPrivate()
{
auto geoObjectIndexFuture =
std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm);
AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
m_geoObjectsKv, m_regionInfoGetter, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount);
LOG(LINFO, ("Geo objects with addresses were built."));
auto geoObjectIndex = geoObjectIndexFuture.get();
LOG(LINFO, ("Index was built."));
if (!geoObjectIndex)
return false;
GeoObjectMaintainer const geoObjectMaintainer{std::move(*geoObjectIndex), m_geoObjectsKv};
LOG(LINFO, ("Enrich address points with outer null building geometry."));
NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry(
geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount);
std::ofstream streamPoiIdsToAddToLocalityIndex(m_pathOutPoiIdsToAddToLocalityIndex);
AddPoisEnrichedWithHouseAddresses(m_geoObjectsKv, geoObjectMaintainer, buildingInfo,
m_pathInGeoObjectsTmpMwm, streamPoiIdsToAddToLocalityIndex,
m_verbose, m_threadsCount);
FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo,
m_threadsCount);
LOG(LINFO, ("Addressless buildings with geometry we used for inner points were filtered"));
LOG(LINFO, ("Geo objects without addresses were built."));
LOG(LINFO, ("Geo objects key-value storage saved to", m_pathOutGeoObjectsKv));
LOG(LINFO, ("Ids of POIs without addresses saved to", m_pathOutPoiIdsToAddToLocalityIndex));
return true;
}
} // namespace geo_objects
} // namespace generator

126
generator/geo_objects/geo_objects.hpp
View file

@ -1,8 +1,11 @@
#pragma once
#include "generator/key_value_storage.hpp"
#include "generator/regions/region_info_getter.hpp"
#include "generator/geo_objects/geo_objects_maintainer.hpp"
#include "geometry/meter.hpp"
#include "geometry/point2d.hpp"
@ -18,6 +21,34 @@ namespace generator
{
namespace geo_objects
{
class RegionInfoGetterProxy
{
public:
using RegionInfoGetter = std::function<boost::optional<KeyValue>(m2::PointD const & pathPoint)>;
RegionInfoGetterProxy(std::string const & pathInRegionsIndex,
std::string const & pathInRegionsKv)
{
m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv);
LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size()));
}
explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter)
: m_externalInfoGetter(std::move(regionInfoGetter))
{
LOG(LINFO, ("External regions info provided"));
}
boost::optional<KeyValue> FindDeepest(m2::PointD const & point) const
{
return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point)
: m_externalInfoGetter->operator()(point);
}
private:
boost::optional<regions::RegionInfoGetter> m_regionInfoGetter;
boost::optional<RegionInfoGetter> m_externalInfoGetter;
};
using IndexReader = ReaderPtr<Reader>;
boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
@ -25,74 +56,35 @@ boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
bool JsonHasBuilding(JsonValue const & json);
class GeoObjectsGenerator
void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
KeyValueStorage & geoObjectsKv, RegionInfoGetterProxy const & regionInfoGetter,
std::string const & pathInGeoObjectsTmpMwm, bool verbose, size_t threadsCount);
struct NullBuildingsInfo
{
public:
using RegionInfoGetter = std::function<boost::optional<KeyValue>(m2::PointD const & pathPoint)>;
class RegionInfoGetterProxy
{
public:
RegionInfoGetterProxy(std::string const & pathInRegionsIndex,
std::string const & pathInRegionsKv)
{
m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv);
LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size()));
}
explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter)
: m_externalInfoGetter(std::move(regionInfoGetter))
{
LOG(LINFO, ("External regions info provided"));
}
boost::optional<KeyValue> FindDeepest(m2::PointD const & point) const
{
return m_regionInfoGetter ? m_regionInfoGetter->FindDeepest(point)
: m_externalInfoGetter->operator()(point);
}
private:
boost::optional<regions::RegionInfoGetter> m_regionInfoGetter;
boost::optional<RegionInfoGetter> m_externalInfoGetter;
};
GeoObjectsGenerator(std::string pathInRegionsIndex, std::string pathInRegionsKv,
std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);
GeoObjectsGenerator(RegionInfoGetter && regionInfoGetter, std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv,
bool verbose, size_t threadsCount);
// This function generates key-value pairs for geo objects.
// First, we try to generate key-value pairs only for houses, since we cannot say anything about
// poi. In this step, we need key-value pairs for the regions and the index for the regions. Then
// we build an index for houses. And then we finish building key-value pairs for poi using this
// index for houses.
bool GenerateGeoObjects();
KeyValueStorage const & GetKeyValueStorage() const
{
return m_geoObjectsKv;
}
private:
bool GenerateGeoObjectsPrivate();
static KeyValueStorage InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv)
{
Platform().RemoveFileIfExists(pathOutGeoObjectsKv);
return KeyValueStorage(pathOutGeoObjectsKv, 0 /* cacheValuesCountLimit */);
}
std::string m_pathInGeoObjectsTmpMwm;
std::string m_pathOutPoiIdsToAddToLocalityIndex;
std::string m_pathOutGeoObjectsKv;
bool m_verbose = false;
size_t m_threadsCount = 1;
KeyValueStorage m_geoObjectsKv;
RegionInfoGetterProxy m_regionInfoGetter;
std::unordered_map<base::GeoObjectId, base::GeoObjectId> m_addressPoints2Buildings;
// Quite possible to have many points for one building. We want to use
// their addresses for POIs according to buildings and have no idea how to distinguish between
// them, so take one random
std::unordered_map<base::GeoObjectId, base::GeoObjectId> m_Buildings2AddressPoint;
};
NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(
GeoObjectMaintainer const & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount);
void AddPoisEnrichedWithHouseAddresses(KeyValueStorage & geoObjectsKv,
GeoObjectMaintainer const & geoObjectMaintainer,
NullBuildingsInfo const & buildingsInfo,
std::string const & pathInGeoObjectsTmpMwm,
std::ostream & streamPoiIdsToAddToLocalityIndex,
bool verbose, size_t threadsCount);
void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pathInGeoObjectsTmpMwm,
NullBuildingsInfo const & buildingsInfo,
size_t threadsCount);
} // namespace geo_objects
} // namespace generator

104
generator/geo_objects/geo_objects_generator.cpp Normal file
View file

@ -0,0 +1,104 @@
#include "generator/geo_objects/geo_objects_generator.hpp"
#include "base/logging.hpp"
#include "base/scope_guard.hpp"
#include "base/timer.hpp"
#include <future>
namespace
{
template <class Activist>
auto Measure(std::string activity, Activist && activist)
{
LOG(LINFO, ("Start", activity));
auto timer = base::Timer();
SCOPE_GUARD(_, [&]() { LOG(LINFO, ("Finish", activity, timer.ElapsedSeconds(), "seconds.")); });
return activist();
}
} // namespace
namespace generator
{
namespace geo_objects
{
GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex,
std::string pathInRegionsKv,
std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
, m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress))
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
, m_verbose(verbose)
, m_threadsCount(threadsCount)
, m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv))
, m_regionInfoGetter(pathInRegionsIndex, pathInRegionsKv)
{
}
GeoObjectsGenerator::GeoObjectsGenerator(
RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter, std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
, m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress))
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
, m_verbose(verbose)
, m_threadsCount(threadsCount)
, m_geoObjectsKv(InitGeoObjectsKv(m_pathOutGeoObjectsKv))
, m_regionInfoGetter(std::move(regionInfoGetter))
{
}
bool GeoObjectsGenerator::GenerateGeoObjects()
{
return Measure("generating geo objects", [&]() { return GenerateGeoObjectsPrivate(); });
}
bool GeoObjectsGenerator::GenerateGeoObjectsPrivate()
{
auto geoObjectIndexFuture =
std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm);
AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
m_geoObjectsKv, m_regionInfoGetter, m_pathInGeoObjectsTmpMwm, m_verbose, m_threadsCount);
LOG(LINFO, ("Geo objects with addresses were built."));
auto geoObjectIndex = geoObjectIndexFuture.get();
LOG(LINFO, ("Index was built."));
if (!geoObjectIndex)
return false;
GeoObjectMaintainer const geoObjectMaintainer{std::move(*geoObjectIndex), m_geoObjectsKv};
LOG(LINFO, ("Enrich address points with outer null building geometry."));
NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry(
geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount);
std::ofstream streamPoiIdsToAddToLocalityIndex(m_pathOutPoiIdsToAddToLocalityIndex);
AddPoisEnrichedWithHouseAddresses(m_geoObjectsKv, geoObjectMaintainer, buildingInfo,
m_pathInGeoObjectsTmpMwm, streamPoiIdsToAddToLocalityIndex,
m_verbose, m_threadsCount);
FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo,
m_threadsCount);
LOG(LINFO, ("Addressless buildings with geometry we used for inner points were filtered"));
LOG(LINFO, ("Geo objects without addresses were built."));
LOG(LINFO, ("Geo objects key-value storage saved to", m_pathOutGeoObjectsKv));
LOG(LINFO, ("Ids of POIs without addresses saved to", m_pathOutPoiIdsToAddToLocalityIndex));
return true;
}
} // namespace geo_objects
} // namespace generator

60
generator/geo_objects/geo_objects_generator.hpp Normal file
View file

@ -0,0 +1,60 @@
#include "generator/key_value_storage.hpp"
#include "generator/regions/region_info_getter.hpp"
#include "generator/geo_objects/geo_objects.hpp"
#include "geometry/meter.hpp"
#include "geometry/point2d.hpp"
#include "platform/platform.hpp"
#include <string>
namespace generator
{
namespace geo_objects
{
class GeoObjectsGenerator
{
public:
GeoObjectsGenerator(std::string pathInRegionsIndex, std::string pathInRegionsKv,
std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);
GeoObjectsGenerator(RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter,
std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);
// This function generates key-value pairs for geo objects.
// First, we try to generate key-value pairs only for houses, since we cannot say anything about
// poi. In this step, we need key-value pairs for the regions and the index for the regions. Then
// we build an index for houses. And then we finish building key-value pairs for poi using this
// index for houses.
bool GenerateGeoObjects();
KeyValueStorage const & GetKeyValueStorage() const
{
return m_geoObjectsKv;
}
private:
bool GenerateGeoObjectsPrivate();
static KeyValueStorage InitGeoObjectsKv(std::string const & pathOutGeoObjectsKv)
{
Platform().RemoveFileIfExists(pathOutGeoObjectsKv);
return KeyValueStorage(pathOutGeoObjectsKv, 0 /* cacheValuesCountLimit */);
}
std::string m_pathInGeoObjectsTmpMwm;
std::string m_pathOutPoiIdsToAddToLocalityIndex;
std::string m_pathOutGeoObjectsKv;
bool m_verbose = false;
size_t m_threadsCount = 1;
KeyValueStorage m_geoObjectsKv;
RegionInfoGetterProxy m_regionInfoGetter;
};
} // namespace geo_objects
} // namespace generator

2
generator/geo_objects/geo_object_maintainer.cpp → generator/geo_objects/geo_objects_maintainer.cpp
View file

@ -1,4 +1,4 @@
#include "generator/geo_objects/geo_object_maintainer.hpp"
#include "generator/geo_objects/geo_objects_maintainer.hpp"
#include <utility>

4
generator/geo_objects/geo_object_maintainer.hpp → generator/geo_objects/geo_objects_maintainer.hpp
View file

@ -42,6 +42,10 @@ public:
return SearchGeoObjectIdsByPoint(m_index, point);
}
KeyValueStorage const & GetKeyValueStorage() const
{
return m_storage;
}
static std::vector<base::GeoObjectId> SearchGeoObjectIdsByPoint(
indexer::GeoObjectsIndex<IndexReader> const & index, m2::PointD const & point);