organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator:geo_objects] Build temporary index synchronously

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-12-20 10:57:47 +03:00 • committed by Sergey Yershov
parent 04ec6f69bc
commit 1bcdf24d4f
4 changed files with 13 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
generator/generator_tests/geo_objects_tests.cpp
View file

@ -134,7 +134,8 @@ void TestFindReverse(std::vector<OsmElementData> const & osmElements,
std::unique_ptr<GeoObjectsGenerator> geoObjectsGenerator{
TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)};
auto const geoObjectsIndex = MakeTempGeoObjectsIndex(geoObjectsFeatures.GetFullPath());
auto const geoObjectsIndex = MakeTempGeoObjectsIndex(geoObjectsFeatures.GetFullPath(),
1 /* threadsCount */);
TEST(geoObjectsIndex, ("Temporary index build failed"));

4
generator/geo_objects/geo_objects.cpp
View file

@ -369,11 +369,11 @@ bool JsonHasBuilding(JsonValue const & json)
}
boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
std::string const & pathToGeoObjectsTmpMwm)
std::string const & pathToGeoObjectsTmpMwm, unsigned int threadsCount)
{
auto const indexFile = GetPlatform().TmpPathForFile();
SCOPE_GUARD(removeIndexFile, std::bind(Platform::RemoveFileIfExists, std::cref(indexFile)));
if (!GenerateGeoObjectsIndex(indexFile, pathToGeoObjectsTmpMwm, 1))
if (!GenerateGeoObjectsIndex(indexFile, pathToGeoObjectsTmpMwm, threadsCount))
{
LOG(LCRITICAL, ("Error generating geo objects index."));
return {};

2
generator/geo_objects/geo_objects.hpp
View file

@ -26,7 +26,7 @@ using IndexReader = ReaderPtr<Reader>;
using RegionInfoLocater = std::function<boost::optional<KeyValue>(m2::PointD const & pathPoint)>;
boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
std::string const & pathToGeoObjectsTmpMwm);
std::string const & pathToGeoObjectsTmpMwm, unsigned int threadsCount);
bool JsonHasBuilding(JsonValue const & json);

22
generator/geo_objects/geo_objects_generator.cpp
View file

@ -46,26 +46,21 @@ bool GeoObjectsGenerator::GenerateGeoObjects()
bool GeoObjectsGenerator::GenerateGeoObjectsPrivate()
{
auto geoObjectIndexFuture =
std::async(std::launch::async, MakeTempGeoObjectsIndex, m_pathInGeoObjectsTmpMwm);
// Index buidling requires a lot of memory (~140GB).
// Build index when there is a lot of memory,
// before AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses().
auto geoObjectIndex = MakeTempGeoObjectsIndex(m_pathInGeoObjectsTmpMwm, m_threadsCount);
if (!geoObjectIndex)
return false;
LOG(LINFO, ("Index was built."));
m_geoObjectMaintainer.SetIndex(std::move(*geoObjectIndex));
AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
m_pathOutGeoObjectsKv, m_geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_regionInfoLocater,
m_verbose, m_threadsCount);
LOG(LINFO, ("Geo objects with addresses were built."));
auto geoObjectIndex = geoObjectIndexFuture.get();
LOG(LINFO, ("Index was built."));
if (!geoObjectIndex)
return false;
m_geoObjectMaintainer.SetIndex(std::move(*geoObjectIndex));
LOG(LINFO, ("Enrich address points with outer null building geometry."));
NullBuildingsInfo const & buildingInfo = EnrichPointsWithOuterBuildingGeometry(
m_geoObjectMaintainer, m_pathInGeoObjectsTmpMwm, m_threadsCount);
@ -77,7 +72,6 @@ bool GeoObjectsGenerator::GenerateGeoObjectsPrivate()
FilterAddresslessThanGaveTheirGeometryToInnerPoints(m_pathInGeoObjectsTmpMwm, buildingInfo,
m_threadsCount);
LOG(LINFO, ("Addressless buildings with geometry we used for inner points were filtered"));
LOG(LINFO, ("Geo objects without addresses were built."));