organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Optimize region's info getter: exclude conversion string -> JSON

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-12-13 14:17:48 +03:00 • committed by Sergey Yershov
parent 33467bf76b
commit 7e785156e9
7 changed files with 13 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
generator/generator_tests/geo_objects_tests.cpp
View file

@ -154,7 +154,7 @@ void TestFindReverse(std::vector<OsmElementData> const & osmElements,
TEST(JsonHasBuilding(JsonValue{std::move(json)}), ("No address for", id));
}
KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath(), 0 /*cacheValuesCountLimit*/};
KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath()};
for (GeoObjectId id : toCheck)
{
@ -241,7 +241,7 @@ void TestPoiHasAddress(std::vector<OsmElementData> const & osmElements)
std::unique_ptr<GeoObjectsGenerator> geoObjectsGenerator = {
TearUp(osmElements, geoObjectsFeatures, idsWithoutAddresses, geoObjectsKeyValue)};
KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath(), 0 /*cacheValuesCountLimit*/};
KeyValueStorage kvStorage{geoObjectsKeyValue.GetFullPath()};
for (GeoObjectId id : expectedIds)
{

2
generator/generator_tests/streets_builder_tests.cpp
View file

@ -74,7 +74,7 @@ UNIT_TEST(StreetsBuilderTest_AggregatedStreetsInKv)
streetsBuilder.SaveStreetsKv(RussiaGetter, streetsJsonlStream);
streetsJsonlStream.flush();
KeyValueStorage streetsStorage{streetsJsonlFile.GetFullPath(), 0 /* cacheValuesCountLimit */};
KeyValueStorage streetsStorage{streetsJsonlFile.GetFullPath()};
TEST_EQUAL(streetsStorage.Size(), 2, ());
TEST(bool(streetsStorage.Find(MakeOsmWay(1).GetEncodedId())) !=
bool(streetsStorage.Find(MakeOsmWay(2).GetEncodedId())),

4
generator/geo_objects/geo_objects_maintainer.cpp
View file

@ -73,7 +73,7 @@ base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObject(
continue;
auto regionJsonValue = m_regionIdGetter(geoData.m_regionId);
auto const & regionJsonValue = m_regionIdGetter(geoData.m_regionId);
if (!regionJsonValue)
return {};
@ -92,7 +92,7 @@ base::JSONPtr GeoObjectMaintainer::GeoObjectsView::GetFullGeoObjectWithoutNameAn
return {};
auto const geoData = it->second;
auto regionJsonValue = m_regionIdGetter(geoData.m_regionId);
auto const & regionJsonValue = m_regionIdGetter(geoData.m_regionId);
if (!regionJsonValue)
return {};

21
generator/key_value_storage.cpp
View file

@ -12,9 +12,7 @@
namespace generator
{
KeyValueStorage::KeyValueStorage(std::string const & path, size_t cacheValuesCountLimit,
std::function<bool(KeyValue const &)> const & pred)
: m_cacheValuesCountLimit{cacheValuesCountLimit}
KeyValueStorage::KeyValueStorage(std::string const & path)
{
auto storage = std::ifstream{path};
std::string line;
@ -39,13 +37,7 @@ KeyValueStorage::KeyValueStorage(std::string const & path, size_t cacheValuesCou
continue;
}
if (!pred({key, json}))
continue;
if (m_cacheValuesCountLimit <= m_values.size())
m_values.emplace(key, std::move(value));
else
m_values.emplace(key, std::move(json));
m_values.emplace(key, std::move(json));
}
}
@ -96,14 +88,7 @@ std::shared_ptr<JsonValue> KeyValueStorage::Find(uint64_t key) const
if (it == std::end(m_values))
return {};
if (auto json = boost::get<std::shared_ptr<JsonValue>>(&it->second))
return *json;
auto const & jsonString = boost::get<std::string>(it->second);
auto json = std::make_shared<JsonValue>(base::LoadFromString(jsonString));
CHECK(json, ());
return json;
return it->second;
}
std::string KeyValueStorage::SerializeDref(uint64_t number)

9
generator/key_value_storage.hpp
View file

@ -44,8 +44,7 @@ public:
// https://jira.mail.ru/browse/MAPSB2B-41
static uint32_t constexpr kDefaultPrecision = 9;
explicit KeyValueStorage(std::string const & kvPath, size_t cacheValuesCountLimit,
std::function<bool(KeyValue const &)> const & pred = DefaultPred);
explicit KeyValueStorage(std::string const & kvPath);
KeyValueStorage(KeyValueStorage &&) = default;
KeyValueStorage & operator=(KeyValueStorage &&) = default;
@ -67,12 +66,8 @@ public:
static std::string SerializeDref(uint64_t number);
private:
using Value = boost::variant<std::shared_ptr<JsonValue>, std::string>;
static bool DefaultPred(KeyValue const &) { return true; }
static bool ParseKeyValueLine(std::string const & line, std::streamoff lineNumber, uint64_t & key,
std::string & value);
std::unordered_map<uint64_t, Value> m_values;
size_t m_cacheValuesCountLimit;
std::unordered_map<uint64_t, std::shared_ptr<JsonValue>> m_values;
};
} // namespace generator

4
generator/regions/region_info_getter.cpp
View file

@ -10,7 +10,7 @@ namespace regions
{
RegionInfoGetter::RegionInfoGetter(std::string const & indexPath, std::string const & kvPath)
: m_index{indexer::ReadIndex<indexer::RegionsIndexBox<IndexReader>, MmapReader>(indexPath)}
, m_storage(kvPath, 1'000'000)
, m_storage(kvPath)
{
m_borders.Deserialize(indexPath);
}
@ -44,7 +44,7 @@ boost::optional<KeyValue> RegionInfoGetter::GetDeepest(m2::PointD const & point,
std::multimap<int, KeyValue> regionsByRank;
for (auto const & id : ids)
{
auto const region = m_storage.Find(id.GetEncodedId());
auto const & region = m_storage.Find(id.GetEncodedId());
if (!region)
{
LOG(LWARNING, ("Id not found in region key-value storage:", id));

2
generator/streets/streets_builder.cpp
View file

@ -133,7 +133,7 @@ void StreetsBuilder::SaveStreetsKv(RegionGetter const & regionGetter,
{
for (auto const & region : m_regions)
{
auto const && regionObject = regionGetter(region.first);
auto const & regionObject = regionGetter(region.first);
CHECK(regionObject, ());
SaveRegionStreetsKv(region.second, region.first, *regionObject, streamStreetsKv);
}