pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] Review notes

Browse source
This commit is contained in:
LaGrunge 2019-08-12 20:13:14 +03:00 • committed by cc-engineering
parent e946c8a4a5
commit e14930a6a5
6 changed files with 54 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
generator/generator_tests/geo_objects_tests.cpp
View file

@ -98,11 +98,14 @@ std::unique_ptr<GeoObjectsGenerator> TearUp(std::vector<OsmElementData> const &
}
})"));
auto regionGetter = [value](auto && point) { return KeyValue{1, value}; };
return std::make_unique<GeoObjectsGenerator>(
regionGetter, geoObjectsFeatures.GetFullPath(), idsWithoutAddresses.GetFullPath(),
auto regionInfoGetter = [value](auto && point) { return KeyValue{1, value}; };
auto regionIdGetter = [value](auto && point) { return value; };
auto result = std::make_unique<GeoObjectsGenerator>(
regionInfoGetter, regionIdGetter, geoObjectsFeatures.GetFullPath(), idsWithoutAddresses.GetFullPath(),
geoObjectsKeyValue.GetFullPath(), false /* verbose */, 1 /* threadsCount */);
TEST(result->GenerateGeoObjects(), ("Generate Geo Objects failed"));
return result;
}
void TestRegionAddress(json_t const * json)
@ -130,8 +133,6 @@ void TestFindReverse(std::vector<OsmElementData> const & osmElements,
std::unique_ptr<GeoObjectsGenerator> geoObjectsGenerator{
TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)};
TEST(geoObjectsGenerator->GenerateGeoObjects(), ("Generate Geo Objects failed"));
auto const geoObjectsIndex = MakeTempGeoObjectsIndex(geoObjectsFeatures.GetFullPath());
TEST(geoObjectsIndex.has_value(), ("Temporary index build failed"));
@ -144,11 +145,10 @@ void TestFindReverse(std::vector<OsmElementData> const & osmElements,
auto const & view = geoObjectsGenerator->GetMaintainer().CreateView();
const auto & toCheck = expected.empty() ? expectedIds : expected;
std::cerr << DebugPrint(toCheck) << std::endl;
for (auto const & id : toCheck)
{
auto json = view.GetFullGeoObject(id);
auto json = view.GetFullGeoObjectWithoutNameAndCoordinates(id);
TestRegionAddress(json.get());
TEST(JsonHasBuilding(JsonValue{std::move(json)}), ("No address for", id));
}
@ -240,8 +240,6 @@ void TestPoiHasAddress(std::vector<OsmElementData> const & osmElements)
std::unique_ptr<GeoObjectsGenerator> geoObjectsGenerator = {
TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)};
TEST(geoObjectsGenerator->GenerateGeoObjects(), ("Generate Geo Objects failed"));
geoObjectsGenerator.reset(nullptr);
KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath(), 0 /*cacheValuesCountLimit*/};

16
generator/geo_objects/geo_objects.cpp
View file

@ -107,7 +107,6 @@ BuildingsGeometries GetBuildingsGeometry(std::string const & pathInGeoObjectsTmp
};
ForEachParallelFromDatRawFormat(threadsCount, pathInGeoObjectsTmpMwm, saveIdFold);
LOG(LINFO, (sizeof(result), "is size of geometries in bytes"));
return result;
}
@ -160,26 +159,26 @@ size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjects
return pointsEnriched;
}
base::JSONPtr FindHouse(FeatureBuilder const & fb, GeoObjectMaintainer & geoObjectMaintainer,
base::JSONPtr FindHouse(FeatureBuilder const & fb,
GeoObjectMaintainer::GeoObjectsView const & geoView,
NullBuildingsInfo const & buildingsInfo)
{
auto const & view = geoObjectMaintainer.CreateView();
base::JSONPtr house =
view.GetFullGeoObject(fb.GetKeyPoint(), [](GeoObjectMaintainer::GeoObjectData const & data) {
geoView.GetFullGeoObject(fb.GetKeyPoint(), [](GeoObjectMaintainer::GeoObjectData const & data) {
return !data.m_house.empty();
});
if (house)
return house;
std::vector<base::GeoObjectId> potentialIds = view.SearchObjectsInIndex(fb.GetKeyPoint());
std::vector<base::GeoObjectId> potentialIds = geoView.SearchObjectsInIndex(fb.GetKeyPoint());
for (base::GeoObjectId id : potentialIds)
{
auto const it = buildingsInfo.m_Buildings2AddressPoint.find(id);
if (it != buildingsInfo.m_Buildings2AddressPoint.end())
return view.GetFullGeoObject(it->second);
return geoView.GetFullGeoObjectWithoutNameAndCoordinates(it->second);
}
return {};
@ -271,13 +270,16 @@ void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer
{
std::atomic_size_t counter{0};
auto const & view = geoObjectMaintainer.CreateView();
auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) {
if (!GeoObjectsFilter::IsPoi(fb))
return;
if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb))
return;
auto house = FindHouse(fb, geoObjectMaintainer, buildingsInfo);
// No name and coordinates here, we will take it from fb in MakeJsonValueWithNameFromFeature
auto house = FindHouse(fb, view, buildingsInfo);
if (!house)
return;

6
generator/geo_objects/geo_objects_generator.cpp
View file

@ -40,7 +40,8 @@ GeoObjectsGenerator::GeoObjectsGenerator(std::string pathInRegionsIndex,
}
GeoObjectsGenerator::GeoObjectsGenerator(
RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter, std::string pathInGeoObjectsTmpMwm,
RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter,
RegionInfoGetterProxy::RegionIdGetter && regionIdGetter, std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
@ -48,7 +49,8 @@ GeoObjectsGenerator::GeoObjectsGenerator(
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
, m_verbose(verbose)
, m_threadsCount(threadsCount)
, m_geoObjectMaintainer{m_pathOutGeoObjectsKv, RegionInfoGetterProxy(std::move(regionInfoGetter))}
, m_geoObjectMaintainer{m_pathOutGeoObjectsKv, RegionInfoGetterProxy(std::move(regionInfoGetter),
std::move(regionIdGetter))}
{
}

1
generator/geo_objects/geo_objects_generator.hpp
View file

@ -23,6 +23,7 @@ public:
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);
GeoObjectsGenerator(RegionInfoGetterProxy::RegionInfoGetter && regionInfoGetter,
RegionInfoGetterProxy::RegionIdGetter && regionIdGetter,
std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);

26
generator/geo_objects/geo_objects_maintainer.cpp
View file

@ -1,6 +1,7 @@
#include "generator/geo_objects/geo_objects_maintainer.hpp"
#include "generator/geo_objects/geo_objects_filter.hpp"
#include "generator/key_value_storage.hpp"
#include "generator/translation.hpp"
#include <utility>
@ -88,7 +89,7 @@ void GeoObjectMaintainer::StoreAndEnrich(feature::FeatureBuilder & fb)
auto const it = m_geoId2GeoData.emplace(
std::make_pair(id, GeoObjectData{fb.GetParams().GetStreet(), fb.GetParams().house.Get(),
fb.GetKeyPoint(), fb.GetMultilangName()}));
base::GeoObjectId(regionKeyValue->first)}));
// Duplicate ID's are possible
if (!it.second)
@ -100,7 +101,7 @@ void GeoObjectMaintainer::StoreAndEnrich(feature::FeatureBuilder & fb)
void GeoObjectMaintainer::WriteToStorage(base::GeoObjectId id, JsonValue && value)
{
std::lock_guard<std::mutex> lock(m_updateMutex);
std::lock_guard<std::mutex> lock(m_storageMutex);
m_geoObjectsKvStorage << KeyValueStorage::SerializeFullLine(id.GetEncodedId(), std::move(value));
}
@ -121,30 +122,33 @@ base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObject(
if (!pred(geoData))
continue;
auto regionKeyValue = m_regionInfoGetter.FindDeepest(geoData.m_keyPoint);
if (!regionKeyValue)
auto regionJsonValue = m_regionInfoGetter.FindById(geoData.m_regionId);
if (!regionJsonValue)
return {};
return AddAddress(geoData.m_street, geoData.m_house, geoData.m_keyPoint, geoData.m_name,
*regionKeyValue);
return AddAddress(geoData.m_street, geoData.m_house, point, StringUtf8Multilang(),
KeyValue(geoData.m_regionId.GetEncodedId(), regionJsonValue));
}
return {};
}
base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObject(base::GeoObjectId id) const
base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObjectWithoutNameAndCoordinates(
base::GeoObjectId id) const
{
auto const it = m_geoId2GeoData.find(id);
if (it == m_geoId2GeoData.end())
return {};
auto const geoData = it->second;
auto regionKeyValue = m_regionInfoGetter.FindDeepest(geoData.m_keyPoint);
if (!regionKeyValue)
auto regionJsonValue = m_regionInfoGetter.FindById(geoData.m_regionId);
if (!regionJsonValue)
return {};
return AddAddress(geoData.m_street, geoData.m_house, geoData.m_keyPoint, geoData.m_name,
*regionKeyValue);
// no need to store name here, it will be overriden by poi name
return AddAddress(geoData.m_street, geoData.m_house, m2::PointD(), StringUtf8Multilang(),
KeyValue(geoData.m_regionId.GetEncodedId(), regionJsonValue));
}
boost::optional<GeoObjectMaintainer::GeoObjectData> GeoObjectMaintainer::GeoObjectsView::GetGeoData(

21
generator/geo_objects/geo_objects_maintainer.hpp
View file

@ -31,14 +31,18 @@ class RegionInfoGetterProxy
{
public:
using RegionInfoGetter = std::function<boost::optional<KeyValue>(m2::PointD const & pathPoint)>;
using RegionIdGetter = std::function<std::shared_ptr<JsonValue>(base::GeoObjectId id)>;
RegionInfoGetterProxy(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv)
{
m_regionInfoGetter = regions::RegionInfoGetter(pathInRegionsIndex, pathInRegionsKv);
LOG(LINFO, ("Size of regions key-value storage:", m_regionInfoGetter->GetStorage().Size()));
}
explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter)
explicit RegionInfoGetterProxy(RegionInfoGetter && regionInfoGetter, RegionIdGetter && regionIdGetter)
: m_externalInfoGetter(std::move(regionInfoGetter))
, m_externalRegionIdGetter(std::move(regionIdGetter))
{
LOG(LINFO, ("External regions info provided"));
}
@ -49,9 +53,18 @@ public:
: m_externalInfoGetter->operator()(point);
}
std::shared_ptr<JsonValue> FindById(base::GeoObjectId id) const
{
return m_externalRegionIdGetter ? m_externalRegionIdGetter->operator()(id) :
m_regionInfoGetter->GetStorage().Find(id.GetEncodedId());
}
private:
boost::optional<regions::RegionInfoGetter> m_regionInfoGetter;
boost::optional<RegionInfoGetter> m_externalInfoGetter;
boost::optional<RegionIdGetter> m_externalRegionIdGetter;
};
void UpdateCoordinates(m2::PointD const & point, base::JSONPtr & json);
@ -63,8 +76,7 @@ public:
{
std::string m_street;
std::string m_house;
m2::PointD m_keyPoint;
StringUtf8Multilang m_name;
base::GeoObjectId m_regionId;
};
using GeoId2GeoData = std::unordered_map<base::GeoObjectId, GeoObjectData>;
@ -92,7 +104,7 @@ public:
return SearchGeoObjectIdsByPoint(m_geoIndex, point);
}
base::JSONPtr GetFullGeoObject(base::GeoObjectId id) const;
base::JSONPtr GetFullGeoObjectWithoutNameAndCoordinates(base::GeoObjectId id) const;
base::JSONPtr GetFullGeoObject(
m2::PointD point,
@ -128,6 +140,7 @@ private:
std::fstream m_geoObjectsKvStorage;
std::mutex m_updateMutex;
std::mutex m_storageMutex;
GeoIndex m_index;
RegionInfoGetterProxy m_regionInfoGetter;