AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[storage][lightweight] added lightweight country info reader.

Browse source
This commit is contained in:
Arsentiy Milchakov 2018-06-19 14:16:22 +03:00 • committed by Vlad Mihaylenko
parent 360fdd2db0
commit c701d2d7be
9 changed files with 305 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
map/framework_light.hpp
View file

@ -5,8 +5,14 @@
#include "ugc/storage.hpp"
#include "storage/country_info_reader_light.hpp"
#include "geometry/point2d.hpp"
#include "base/assert.hpp"
#include <memory>
namespace lightweight
{
struct LightFrameworkTest;
@ -18,6 +24,10 @@ enum RequestType
REQUEST_TYPE_USER_AUTH_STATUS = 1u << 1,
REQUEST_TYPE_NUMBER_OF_UNSENT_EDITS = 1u << 2,
REQUEST_TYPE_BOOKMARKS_CLOUD_ENABLED = 1u << 3,
// Be careful to use this flag. Loading with this flag can produces a hard pressure on the disk
// and takes much time. For example it takes ~50ms on LG Nexus 5, ~100ms on Samsung A5, ~200ms on
// Fly IQ4403.
REQUEST_TYPE_LOCATION = 1u << 4,
};
using RequestTypeMask = unsigned;
@ -59,18 +69,28 @@ public:
request ^= REQUEST_TYPE_BOOKMARKS_CLOUD_ENABLED;
}
if (request & REQUEST_TYPE_LOCATION)
{
m_countryInfoReader = std::make_unique<CountryInfoReader>();
request ^= REQUEST_TYPE_LOCATION;
}
CHECK_EQUAL(request, REQUEST_TYPE_EMPTY, ("Incorrect mask type:", request));
}
template <RequestTypeMask Type>
auto Get() const;
template <RequestTypeMask Type>
auto Get(m2::PointD const & pt) const;
private:
RequestTypeMask m_request;
bool m_userAuthStatus = false;
size_t m_numberOfUnsentUGC = 0;
size_t m_numberOfUnsentEdits = 0;
bool m_bookmarksCloudEnabled = false;
std::unique_ptr<CountryInfoReader> m_countryInfoReader;
};
template<>
@ -100,4 +120,14 @@ auto Framework::Get<REQUEST_TYPE_BOOKMARKS_CLOUD_ENABLED>() const
ASSERT(m_request & REQUEST_TYPE_BOOKMARKS_CLOUD_ENABLED, (m_request));
return m_bookmarksCloudEnabled;
}
template <>
auto Framework::Get<REQUEST_TYPE_LOCATION>(m2::PointD const & pt) const
{
ASSERT(m_request & REQUEST_TYPE_LOCATION, (m_request));
CHECK(m_countryInfoReader, ());
return m_countryInfoReader->GetMwmInfo(pt);
}
} // namespace lightweight

2
storage/CMakeLists.txt
View file

@ -13,6 +13,8 @@ set(
country_info_getter.hpp
country_name_getter.cpp
country_name_getter.hpp
country_info_reader_light.cpp
country_info_reader_light.hpp
country_parent_getter.cpp
country_parent_getter.hpp
country_polygon.hpp

90
storage/country_info_getter.cpp
View file

@ -50,13 +50,59 @@ private:
};
} // namespace
// CountryInfoGetter -------------------------------------------------------------------------------
TCountryId CountryInfoGetter::GetRegionCountryId(m2::PointD const & pt) const
// CountryInfoGetterBase ---------------------------------------------------------------------------------
TCountryId CountryInfoGetterBase::GetRegionCountryId(m2::PointD const & pt) const
{
TRegionId const id = FindFirstCountry(pt);
return id != kInvalidId ? m_countries[id].m_countryId : kInvalidCountryId;
}
bool CountryInfoGetterBase::IsBelongToRegions(m2::PointD const & pt,
TRegionIdSet const & regions) const
{
for (auto const & id : regions)
{
if (m_countries[id].m_rect.IsPointInside(pt) && IsBelongToRegionImpl(id, pt))
return true;
}
return false;
}
bool CountryInfoGetterBase::IsBelongToRegions(TCountryId const & countryId,
TRegionIdSet const & regions) const
{
for (auto const & id : regions)
{
if (m_countries[id].m_countryId == countryId)
return true;
}
return false;
}
void CountryInfoGetterBase::RegionIdsToCountryIds(TRegionIdSet const & regions,
TCountriesVec & countries) const
{
for (auto const & id : regions)
countries.push_back(m_countries[id].m_countryId);
}
CountryInfoGetterBase::TRegionId CountryInfoGetterBase::FindFirstCountry(m2::PointD const & pt) const
{
for (size_t id = 0; id < m_countries.size(); ++id)
{
if (m_countries[id].m_rect.IsPointInside(pt) && IsBelongToRegionImpl(id, pt))
return id;
}
ms::LatLon const latLon = MercatorBounds::ToLatLon(pt);
alohalytics::LogEvent(m_isSingleMwm
? "Small mwm case. CountryInfoGetter could not find any mwm by point."
: "Big mwm case. CountryInfoGetter could not find any mwm by point.",
alohalytics::Location::FromLatLon(latLon.lat, latLon.lon));
return kInvalidId;
}
// CountryInfoGetter -------------------------------------------------------------------------------
vector<TCountryId> CountryInfoGetter::GetRegionsCountryIdByRect(m2::RectD const & rect, bool rough) const
{
size_t constexpr kAverageSize = 10;
@ -149,52 +195,12 @@ void CountryInfoGetter::GetMatchedRegions(string const & affiliation, TRegionIdS
regions.push_back(i);
}
}
bool CountryInfoGetter::IsBelongToRegions(m2::PointD const & pt, TRegionIdSet const & regions) const
{
for (auto const & id : regions)
{
if (m_countries[id].m_rect.IsPointInside(pt) && IsBelongToRegionImpl(id, pt))
return true;
}
return false;
}
bool CountryInfoGetter::IsBelongToRegions(TCountryId const & countryId, TRegionIdSet const & regions) const
{
for (auto const & id : regions)
{
if (m_countries[id].m_countryId == countryId)
return true;
}
return false;
}
void CountryInfoGetter::RegionIdsToCountryIds(TRegionIdSet const & regions, TCountriesVec & countries) const
{
for (auto const & id : regions)
countries.push_back(m_countries[id].m_countryId);
}
void CountryInfoGetter::InitAffiliationsInfo(TMappingAffiliations const * affiliations)
{
m_affiliations = affiliations;
}
CountryInfoGetter::TRegionId CountryInfoGetter::FindFirstCountry(m2::PointD const & pt) const
{
for (size_t id = 0; id < m_countries.size(); ++id)
{
if (m_countries[id].m_rect.IsPointInside(pt) && IsBelongToRegionImpl(id, pt))
return id;
}
ms::LatLon const latLon = MercatorBounds::ToLatLon(pt);
alohalytics::LogEvent(m_isSingleMwm ? "Small mwm case. CountryInfoGetter could not find any mwm by point."
: "Big mwm case. CountryInfoGetter could not find any mwm by point.",
alohalytics::Location::FromLatLon(latLon.lat, latLon.lon));
return kInvalidId;
}
template <typename ToDo>
void CountryInfoGetter::ForEachCountry(string const & prefix, ToDo && toDo) const

75
storage/country_info_getter.hpp
View file

@ -20,25 +20,57 @@
namespace storage
{
// This class allows users to get information about country by point
// or by name.
//
// *NOTE* This class is thread-safe.
class CountryInfoGetter
class CountryInfoGetterBase
{
public:
// Identifier of a region (index in m_countries array).
using TRegionId = size_t;
using TRegionIdSet = std::vector<TRegionId>;
CountryInfoGetter(bool isSingleMwm) : m_isSingleMwm(isSingleMwm) {}
virtual ~CountryInfoGetter() = default;
CountryInfoGetterBase(bool isSingleMwm) : m_isSingleMwm(isSingleMwm) {}
virtual ~CountryInfoGetterBase() = default;
// Returns country file name without an extension for a country |pt|
// belongs to. If there is no such country, returns an empty
// string.
TCountryId GetRegionCountryId(m2::PointD const & pt) const;
// Returns true when |pt| belongs to at least one of the specified
// |regions|.
bool IsBelongToRegions(m2::PointD const & pt, TRegionIdSet const & regions) const;
// Returns true if there're at least one region with id equals to
// |countryId|.
bool IsBelongToRegions(TCountryId const & countryId, TRegionIdSet const & regions) const;
void RegionIdsToCountryIds(TRegionIdSet const & regions, TCountriesVec & countries) const;
protected:
// Returns identifier of a first country containing |pt|.
TRegionId FindFirstCountry(m2::PointD const & pt) const;
// Returns true when |pt| belongs to a country identified by |id|.
virtual bool IsBelongToRegionImpl(size_t id, m2::PointD const & pt) const = 0;
// @TODO(bykoianko): consider to get rid of m_countryIndex.
// The possibility should be considered.
// List of all known countries.
std::vector<CountryDef> m_countries;
// m_isSingleMwm == true if the system is currently working with single (small) mwms
// and false otherwise.
// @TODO(bykoianko) Init m_isSingleMwm correctly.
bool m_isSingleMwm;
};
// This class allows users to get information about country by point
// or by name.
//
// *NOTE* This class is thread-safe.
class CountryInfoGetter : public CountryInfoGetterBase
{
public:
CountryInfoGetter(bool isSingleMwm) : CountryInfoGetterBase(isSingleMwm) {}
// Returns vector of countries file names without an extension for
// countries belong to |rect|. |rough| provides fast rough result
// or a slower but more precise one.
@ -73,27 +105,13 @@ public:
// Returns identifiers for all regions matching to correspondent |affiliation|.
virtual void GetMatchedRegions(string const & affiliation, TRegionIdSet & regions) const;
// Returns true when |pt| belongs to at least one of the specified
// |regions|.
bool IsBelongToRegions(m2::PointD const & pt, TRegionIdSet const & regions) const;
// Returns true if there're at least one region with id equals to
// |countryId|.
bool IsBelongToRegions(TCountryId const & countryId, TRegionIdSet const & regions) const;
void RegionIdsToCountryIds(TRegionIdSet const & regions, TCountriesVec & countries) const;
// Clears regions cache.
inline void ClearCaches() const { ClearCachesImpl(); }
void InitAffiliationsInfo(TMappingAffiliations const * affiliations);
protected:
CountryInfoGetter() = default;
// Returns identifier of a first country containing |pt|.
TRegionId FindFirstCountry(m2::PointD const & pt) const;
CountryInfoGetter() : CountryInfoGetterBase(true) {};
// Invokes |toDo| on each country whose name starts with |prefix|.
template <typename ToDo>
void ForEachCountry(string const & prefix, ToDo && toDo) const;
@ -101,19 +119,11 @@ protected:
// Clears regions cache.
virtual void ClearCachesImpl() const = 0;
// Returns true when |pt| belongs to a country identified by |id|.
virtual bool IsBelongToRegionImpl(size_t id, m2::PointD const & pt) const = 0;
// Returns true when |rect| intersects a country identified by |id|.
virtual bool IsIntersectedByRegionImpl(size_t id, m2::RectD const & rect) const = 0;
// Returns true when the distance from |pt| to country identified by |id| less then |distance|.
virtual bool IsCloseEnough(size_t id, m2::PointD const & pt, double distance) = 0;
// @TODO(bykoianko): consider to get rid of m_countryIndex.
// The possibility should be considered.
// List of all known countries.
std::vector<CountryDef> m_countries;
// Maps all leaf country id (file names) to their indices in m_countries.
std::unordered_map<TCountryId, TRegionId> m_countryIndex;
@ -121,11 +131,6 @@ protected:
// Maps country file name without an extension to a country info.
std::map<std::string, CountryInfo> m_id2info;
// m_isSingleMwm == true if the system is currently working with single (small) mwms
// and false otherwise.
// @TODO(bykoianko) Init m_isSingleMwm correctly.
bool m_isSingleMwm;
};
// This class reads info about countries from polygons file and

78
storage/country_info_reader_light.cpp Normal file
View file

@ -0,0 +1,78 @@
#include "storage/country_info_reader_light.hpp"
#include "storage/country_decl.hpp"
#include "storage/country_polygon.hpp"
#include "platform/platform.hpp"
#include "platform/preferred_languages.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include "coding/geometry_coding.hpp"
#include "coding/read_write_utils.hpp"
#include "geometry/region2d.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
#include <chrono>
#include <string>
#include <utility>
#include <vector>
namespace lightweight
{
CountryInfoReader::CountryInfoReader()
: CountryInfoGetterBase(true)
{
try
{
m_reader = std::make_unique<FilesContainerR>(GetPlatform().GetReader(PACKED_POLYGONS_FILE));
ReaderSource<ModelReaderPtr> src(m_reader->GetReader(PACKED_POLYGONS_INFO_TAG));
rw::Read(src, m_countries);
}
catch (FileReader::Exception const & exception)
{
LOG(LERROR,
("Exception while reading file:", PACKED_POLYGONS_FILE, "reason:", exception.what()));
m_reader.reset();
m_countries.clear();
}
m_nameGetter.SetLocale(languages::GetCurrentTwine());
}
bool CountryInfoReader::IsBelongToRegionImpl(size_t id, m2::PointD const & pt) const
{
// Load regions from file.
ReaderSource<ModelReaderPtr> src(m_reader->GetReader(strings::to_string(id)));
uint32_t const count = ReadVarUint<uint32_t>(src);
std::vector<m2::RegionD> regions;
for (size_t i = 0; i < count; ++i)
{
std::vector<m2::PointD> points;
serial::LoadOuterPath(src, serial::GeometryCodingParams(), points);
regions.emplace_back(move(points));
}
for (auto const & region : regions)
{
if (region.Contains(pt))
return true;
}
return false;
}
CountryInfoReader::Info CountryInfoReader::GetMwmInfo(m2::PointD const & pt) const
{
Info info;
info.m_id = GetRegionCountryId(pt);
info.m_name = m_nameGetter(info.m_id);
return info;
}
} // namespace lightweight

36
storage/country_info_reader_light.hpp Normal file
View file

@ -0,0 +1,36 @@
#pragma once
#include "storage/index.hpp"
#include "storage/country_info_getter.hpp"
#include "storage/country_name_getter.hpp"
#include "coding/file_container.hpp"
#include "geometry/point2d.hpp"
#include <cstdint>
#include <memory>
namespace lightweight
{
// Protected inheritance for test purposes only.
class CountryInfoReader : protected storage::CountryInfoGetterBase
{
public:
struct Info
{
storage::TCountryId m_id;
std::string m_name;
};
CountryInfoReader();
Info GetMwmInfo(m2::PointD const & pt) const;
protected:
bool IsBelongToRegionImpl(size_t id, m2::PointD const & pt) const override;
private:
std::unique_ptr<FilesContainerR> m_reader;
storage::CountryNameGetter m_nameGetter;
};
} // namespace lightweight

1
storage/storage_integration_tests/CMakeLists.txt
View file

@ -5,6 +5,7 @@ add_definitions("-DOMIM_UNIT_TEST_WITH_QT_EVENT_LOOP")
set(
SRC
lightweight_matching_tests.cpp
migrate_tests.cpp
storage_3levels_tests.cpp
storage_downloading_tests.cpp

58
storage/storage_integration_tests/lightweight_matching_tests.cpp Normal file
View file

@ -0,0 +1,58 @@
#include "testing/testing.hpp"
#include "storage/country_info_reader_light.hpp"
#include "storage/index.hpp"
#include "geometry/mercator.hpp"
#include "geometry/point2d.hpp"
#include <utility>
#include <vector>
using namespace lightweight;
namespace
{
double constexpr kStepInMercator = 1; // 1 mercator ~= 9602.84 meters
struct PointAndCountry
{
PointAndCountry(m2::PointD && pt, storage::TCountryId && country)
: m_pt(std::move(pt))
, m_country(std::move(country))
{
}
m2::PointD m_pt;
storage::TCountryId m_country;
};
UNIT_CLASS_TEST(CountryInfoReader, LightweightMatching)
{
auto const reader = storage::CountryInfoReader::CreateCountryInfoReader(GetPlatform());
LOG(LINFO, ("Generating dataset..."));
std::vector<PointAndCountry> dataset;
for (auto x = MercatorBounds::minX; x <= MercatorBounds::maxX; x += kStepInMercator)
{
for (auto y = MercatorBounds::minY; y <= MercatorBounds::maxY; y += kStepInMercator)
{
m2::PointD pt(x, y);
dataset.emplace_back(std::move(pt), reader->GetRegionCountryId(pt));
}
}
{
m2::PointD ptFrom = {MercatorBounds::minX, MercatorBounds::minY};
m2::PointD ptTo = {MercatorBounds::minX + kStepInMercator, MercatorBounds::minY};
auto const stepSizeInMeters = MercatorBounds::DistanceOnEarth(ptFrom, ptTo);
LOG(LINFO, ("The dataset is generated. Dataset size:", dataset.size(),
". The step is:", stepSizeInMeters, "meters"));
}
for (auto const & sample : dataset)
{
TEST_EQUAL(GetRegionCountryId(sample.m_pt), sample.m_country, (sample.m_pt));
}
}
} // namespace

12
xcode/storage/storage.xcodeproj/project.pbxproj
View file

@ -17,6 +17,9 @@
34B093261C61F9BA0066F4C3 /* app_store.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 34B093211C61F9BA0066F4C3 /* app_store.hpp */; };
34C9BCFC1C6CCF85000DC38D /* country_name_getter.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 34C9BCFA1C6CCF85000DC38D /* country_name_getter.cpp */; };
34C9BCFD1C6CCF85000DC38D /* country_name_getter.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 34C9BCFB1C6CCF85000DC38D /* country_name_getter.hpp */; };
3DCD414620D80C0900143533 /* country_info_reader_light.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DCD414420D80C0800143533 /* country_info_reader_light.hpp */; };
3DCD414720D80C0900143533 /* country_info_reader_light.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DCD414520D80C0900143533 /* country_info_reader_light.cpp */; };
3DCD414920D80C1B00143533 /* lightweight_matching_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DCD414820D80C1B00143533 /* lightweight_matching_tests.cpp */; };
401ECED41F56C50900DFDF76 /* country_parent_getter.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */; };
401ECED51F56C50900DFDF76 /* country_parent_getter.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 401ECED31F56C50900DFDF76 /* country_parent_getter.hpp */; };
4586D0C71F48126A00DF9CE5 /* diff_manager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4586D0C51F48126A00DF9CE5 /* diff_manager.cpp */; };
@ -198,6 +201,9 @@
34C9BCFB1C6CCF85000DC38D /* country_name_getter.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = country_name_getter.hpp; sourceTree = "<group>"; };
34F5584A1DBF2FC000A4FC11 /* common-debug.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = "common-debug.xcconfig"; path = "../common-debug.xcconfig"; sourceTree = "<group>"; };
34F5584B1DBF2FC000A4FC11 /* common-release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = "common-release.xcconfig"; path = "../common-release.xcconfig"; sourceTree = "<group>"; };
3DCD414420D80C0800143533 /* country_info_reader_light.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = country_info_reader_light.hpp; sourceTree = "<group>"; };
3DCD414520D80C0900143533 /* country_info_reader_light.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = country_info_reader_light.cpp; sourceTree = "<group>"; };
3DCD414820D80C1B00143533 /* lightweight_matching_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = lightweight_matching_tests.cpp; sourceTree = "<group>"; };
401ECED21F56C50900DFDF76 /* country_parent_getter.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = country_parent_getter.cpp; sourceTree = "<group>"; };
401ECED31F56C50900DFDF76 /* country_parent_getter.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = country_parent_getter.hpp; sourceTree = "<group>"; };
4586D0C51F48126A00DF9CE5 /* diff_manager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = diff_manager.cpp; path = diff_scheme/diff_manager.cpp; sourceTree = "<group>"; };
@ -504,6 +510,7 @@
672480071C60CE3D00EDE56A /* storage_integration_tests */ = {
isa = PBXGroup;
children = (
3DCD414820D80C1B00143533 /* lightweight_matching_tests.cpp */,
67239C941CBBDB0E00C530A8 /* download_calc_size_test.cpp */,
56D8CB971CAC17A80003F420 /* test_defines.cpp */,
56D8CB981CAC17A80003F420 /* test_defines.hpp */,
@ -546,6 +553,8 @@
675342E21A3F59EF00A0A8C3 /* storage */ = {
isa = PBXGroup;
children = (
3DCD414520D80C0900143533 /* country_info_reader_light.cpp */,
3DCD414420D80C0800143533 /* country_info_reader_light.hpp */,
F6BC312A2034366100F677FE /* pinger.cpp */,
F6BC31292034366100F677FE /* pinger.hpp */,
56D0E47F1F8E40340084B18C /* routing_helpers.cpp */,
@ -640,6 +649,7 @@
6753430F1A3F5A2600A0A8C3 /* country.hpp in Headers */,
34B093261C61F9BA0066F4C3 /* app_store.hpp in Headers */,
F6BC312B2034366100F677FE /* pinger.hpp in Headers */,
3DCD414620D80C0900143533 /* country_info_reader_light.hpp in Headers */,
6753430A1A3F5A2600A0A8C3 /* country_decl.hpp in Headers */,
678338D61C723B1A00FD6263 /* helpers.hpp in Headers */,
67247FCF1C60BA8A00EDE56A /* fake_map_files_downloader.hpp in Headers */,
@ -822,10 +832,12 @@
675343091A3F5A2600A0A8C3 /* country_decl.cpp in Sources */,
674125211B4C05FA00A3E828 /* queued_country.cpp in Sources */,
F68CC5BE1F38B967007527C7 /* diff_scheme_checker.cpp in Sources */,
3DCD414720D80C0900143533 /* country_info_reader_light.cpp in Sources */,
6753431A1A3F5A2600A0A8C3 /* storage.cpp in Sources */,
675343161A3F5A2600A0A8C3 /* index.cpp in Sources */,
678338D51C723B1A00FD6263 /* helpers.cpp in Sources */,
674125231B4C05FA00A3E828 /* storage_defines.cpp in Sources */,
3DCD414920D80C1B00143533 /* lightweight_matching_tests.cpp in Sources */,
67BE1DC51CD2180D00572709 /* downloading_policy.cpp in Sources */,
678338D41C723B1A00FD6263 /* country_name_getter_test.cpp in Sources */,
F6BC312C2034366100F677FE /* pinger.cpp in Sources */,