[generator:regions] Add Russsia administrative devisions

This commit is contained in:
Anatoly Serdtcev 2019-06-19 11:22:47 +03:00 committed by Tatiana Yan
parent 7f32e8479e
commit 47d1c7884e
10 changed files with 471 additions and 22 deletions

View file

@ -168,6 +168,8 @@ set(SRC
regions/regions_builder.hpp
regions/regions_fixer.cpp
regions/regions_fixer.hpp
regions/specs/rus.cpp
regions/specs/rus.hpp
restriction_collector.cpp
restriction_collector.hpp
restriction_generator.cpp

View file

@ -41,9 +41,9 @@ UNIT_TEST(RegionInfoCollector_Collect)
{
auto const filename = GetFileName();
CollectorRegionInfo regionInfoCollector(filename);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementCity);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementCountry);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementEmpty);
regionInfoCollector.Collect(kOsmElementCity);
regionInfoCollector.Collect(kOsmElementCountry);
regionInfoCollector.Collect(kOsmElementEmpty);
regionInfoCollector.Save();
RegionInfo regionInfo(filename);
@ -86,7 +86,7 @@ UNIT_TEST(RegionInfoCollector_Get)
{
auto const filename = GetFileName();
CollectorRegionInfo regionInfoCollector(filename);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementCity);
regionInfoCollector.Collect(kOsmElementCity);
regionInfoCollector.Save();
RegionInfo regionInfo(filename);
@ -100,8 +100,8 @@ UNIT_TEST(RegionInfoCollector_Exists)
{
auto const filename = GetFileName();
CollectorRegionInfo regionInfoCollector(filename);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementCity);
regionInfoCollector.CollectFeature(kEmptyFeature, kOsmElementCountry);
regionInfoCollector.Collect(kOsmElementCity);
regionInfoCollector.Collect(kOsmElementCountry);
regionInfoCollector.Save();
RegionInfo regionInfo(filename);

View file

@ -2,11 +2,16 @@
#include "generator/feature_builder.hpp"
#include "generator/generator_tests/common.hpp"
#include "generator/osm2type.hpp"
#include "generator/osm_element.hpp"
#include "generator/regions/collector_region_info.hpp"
#include "generator/regions/place_point.hpp"
#include "generator/regions/regions.hpp"
#include "generator/regions/regions_builder.hpp"
#include "indexer/classificator.hpp"
#include "indexer/classificator_loader.hpp"
#include "platform/platform.hpp"
#include "coding/transliteration.hpp"
@ -19,6 +24,7 @@
#include <limits>
#include <memory>
#include <string>
#include <sstream>
#include <utility>
#include <vector>
@ -31,6 +37,7 @@ using namespace base;
namespace
{
using Tags = std::vector<std::pair<std::string, std::string>>;
auto NodeEntry = OsmElement::EntityType::Node;
FeatureBuilder const kEmptyFeature;
@ -46,24 +53,168 @@ OsmElement CreateOsmRelation(uint64_t id, std::string const & adminLevel,
return el;
}
struct TagValue
{
std::string m_key;
std::string m_value;
TagValue(std::string key, std::string value = {})
: m_key{std::move(key)}, m_value{std::move(value)}
{ }
TagValue operator=(std::string const & value) const
{
CHECK(m_value.empty(), ());
return {m_key, value};
}
};
struct OsmElementData
{
uint64_t m_id;
std::vector<TagValue> m_tags;
std::vector<m2::PointD> m_polygon;
std::vector<OsmElement::Member> m_members;
};
OsmElement MakeOsmElement(OsmElementData const & elementData)
{
OsmElement el;
el.m_id = elementData.m_id;
el.m_type = elementData.m_polygon.size() > 1 ? OsmElement::EntityType::Relation
: OsmElement::EntityType::Node;
for (auto const tag : elementData.m_tags)
el.AddTag(tag.m_key, tag.m_value);
el.m_members = elementData.m_members;
return el;
}
void CollectRegionInfo(std::string const & filename, std::vector<OsmElementData> const & testData)
{
CollectorRegionInfo collector(filename);
for (auto const & elementData : testData)
{
auto el = MakeOsmElement(elementData);
collector.Collect(el);
}
collector.Save();
}
void BuildTestData(std::vector<OsmElementData> const & testData,
RegionsBuilder::Regions & regions, PlacePointsMap & placePointsMap,
RegionInfo & collector)
{
for (auto const & elementData : testData)
{
auto el = MakeOsmElement(elementData);
FeatureBuilder fb;
CHECK(elementData.m_polygon.size() == 1 || elementData.m_polygon.size() == 2, ());
if (elementData.m_polygon.size() == 1)
{
fb.SetCenter(elementData.m_polygon[0]);
}
else if (elementData.m_polygon.size() == 2)
{
auto const & p1 = elementData.m_polygon[0];
auto const & p2 = elementData.m_polygon[1];
vector<m2::PointD> poly = {{p1.x, p1.y}, {p1.x, p2.y}, {p2.x, p2.y}, {p2.x, p1.y}, {p1.x, p1.x}};
fb.AddPolygon(poly);
fb.SetHoles({});
fb.SetArea();
}
auto osmId = el.m_type == OsmElement::EntityType::Relation ? MakeOsmRelation(el.m_id)
: MakeOsmNode(el.m_id);
fb.SetOsmId(osmId);
FeatureParams params;
ftype::GetNameAndType(&el, params, [] (uint32_t type) {
return classif().IsTypeValid(type);
});
fb.SetParams(params);
auto const id = fb.GetMostGenericOsmId();
if (elementData.m_polygon.size() == 1)
placePointsMap.emplace(id, PlacePoint{fb, collector.Get(id)});
else
regions.emplace_back(fb, collector.Get(id));
}
}
class ToLabelingStringPolicy : public ToStringPolicyInterface
{
public:
// ToStringPolicyInterface overrides:
std::string ToString(vector<Node::Ptr> const & path) const override
{
CHECK(!path.empty(), ());
std::stringstream stream;
auto i = path.begin();
auto const & countryName = (*i)->GetData().GetName();
CHECK(!countryName.empty(), ());
stream << (*i)->GetData().GetName();
++i;
for (; i != path.end(); ++i)
stream << ", " << GetLabel((*i)->GetData().GetLevel()) << ": " << (*i)->GetData().GetName();
return stream.str();
}
};
std::vector<std::string> GenerateTestRegions(std::vector<OsmElementData> const & testData)
{
classificator::Load();
auto const filename = GetFileName();
CollectRegionInfo(filename, testData);
RegionsBuilder::Regions regions;
PlacePointsMap placePointsMap;
RegionInfo collector(filename);
BuildTestData(testData, regions, placePointsMap, collector);
RegionsBuilder builder(std::move(regions));
std::vector<std::string> kvRegions;
builder.ForEachCountry([&](std::string const & name, Node::PtrList const & outers) {
for (auto const & tree : outers)
{
ForEachLevelPath(tree, [&] (std::vector<Node::Ptr> const & path) {
kvRegions.push_back(ToLabelingStringPolicy{}.ToString(path));
});
}
});
return kvRegions;
}
bool HasName(std::vector<string> const & coll, std::string const & name)
{
auto const end = std::end(coll);
return std::find(std::begin(coll), end, name) != end;
}
bool ContainsSubname(std::vector<string> const & coll, std::string const & name)
{
auto const end = std::end(coll);
auto hasSubname = [&name] (std::string const & item) { return item.find(name) != string::npos; };
return std::find_if(std::begin(coll), end, hasSubname) != end;
}
std::string MakeCollectorData()
{
auto const filename = GetFileName();
CollectorRegionInfo collector(filename);
collector.CollectFeature(kEmptyFeature, CreateOsmRelation(1 /* id */, "2" /* adminLevel */));
collector.CollectFeature(kEmptyFeature, CreateOsmRelation(2 /* id */, "2" /* adminLevel */));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(3 /* id */, "4" /* adminLevel */, "state"));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(4 /* id */, "4" /* adminLevel */, "state"));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(5 /* id */, "4" /* adminLevel */, "state"));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(6 /* id */, "6" /* adminLevel */, "district"));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(7 /* id */, "6" /* adminLevel */, "district"));
collector.CollectFeature(kEmptyFeature,
CreateOsmRelation(8 /* id */, "4" /* adminLevel */, "state"));
collector.Collect(CreateOsmRelation(1 /* id */, "2" /* adminLevel */));
collector.Collect(CreateOsmRelation(2 /* id */, "2" /* adminLevel */));
collector.Collect(CreateOsmRelation(3 /* id */, "4" /* adminLevel */, "state"));
collector.Collect(CreateOsmRelation(4 /* id */, "4" /* adminLevel */, "state"));
collector.Collect(CreateOsmRelation(5 /* id */, "4" /* adminLevel */, "state"));
collector.Collect(CreateOsmRelation(6 /* id */, "6" /* adminLevel */, "district"));
collector.Collect(CreateOsmRelation(7 /* id */, "6" /* adminLevel */, "district"));
collector.Collect(CreateOsmRelation(8 /* id */, "4" /* adminLevel */, "state"));
collector.Save();
return filename;
}
@ -242,6 +393,101 @@ UNIT_TEST(RegionsBuilderTest_GetCountryTrees)
TEST(NameExists(bankOfNames, "Country_1Country_1_Region_5Country_1_Region_5_Subregion_7"), ());
}
// Russia regions tests ----------------------------------------------------------------------------
UNIT_TEST(RegionsBuilderTest_GenerateRusCitySuburb)
{
TagValue const admin{"admin_level"};
TagValue const place{"place"};
TagValue const name{"name"};
TagValue const ba{"boundary", "administrative"};
auto regions = GenerateTestRegions({
{1, {name = u8"Россия", admin = "2", ba}, {{0, 0}, {50, 50}}},
{2, {name = u8"Сибирский федеральный округ", admin = "3", ba}, {{10, 10}, {20, 20}}},
{3, {name = u8"Омская область", admin = "4", ba}, {{12, 12}, {18, 18}}},
{4, {name = u8"Омск", place = "city"}, {{14, 14}, {16, 16}}},
{5, {name = u8"городской округ Омск", admin = "6", ba}, {{14, 14}, {16, 16}},
{{6, NodeEntry, "admin_centre"}}},
{6, {name = u8"Омск", place = "city"}, {{14.5, 14.5}}},
{7, {name = u8"Кировский административный округ", admin = "9", ba}, {{14, 14}, {15, 15}}},
});
/*
TEST(HasName(regions, u8"Россия, region: Омская область, subregion: городской округ Омск, "
u8"locality: Омск"),
());
TEST(HasName(regions, u8"Россия, region: Омская область, subregion: городской округ Омск, "
u8"locality: Омск, suburb: Кировский административный округ"),
());
*/
}
UNIT_TEST(RegionsBuilderTest_GenerateRusMoscowSuburb)
{
TagValue const admin{"admin_level"};
TagValue const place{"place"};
TagValue const name{"name"};
TagValue const ba{"boundary", "administrative"};
auto regions = GenerateTestRegions({
{1, {name = u8"Россия", admin = "2", ba}, {{0, 0}, {50, 50}}},
{2, {name = u8"Центральный федеральный округ", admin = "3", ba}, {{10, 10}, {20, 20}}},
{3, {name = u8"Москва", admin = "4", ba}, {{12, 12}, {18, 18}}},
{4, {name = u8"Москва", place = "city"}, {{12, 12}, {17, 17}}},
{5, {name = u8"Западный административный округ", admin = "5", ba}, {{14, 14}, {16, 16}}},
{6, {name = u8"район Раменки", admin = "8", ba}, {{14, 14}, {15, 15}}, {{7, NodeEntry, "label"}}},
{7, {name = u8"Раменки", place = "suburb"}, {{14.5, 14.5}}}, // label
{8, {name = u8"Тропарёво", place = "suburb"}, {{15.1, 15.1}}}, // no label
{9, {name = u8"Воробъёвы горы", place = "suburb"}, {{14.5, 14.5}, {14.6, 14.6}}},
{10, {name = u8"Центр", place = "suburb"}, {{15, 15}, {15.5, 15.5}}},
});
TEST(HasName(regions, u8"Россия, region: Москва"), ());
TEST(HasName(regions, u8"Россия, region: Москва, subregion: Западный административный округ, "
u8"locality: Москва"),
());
/* FIXME:
TEST(HasName(regions, u8"Россия, region: Москва, subregion: Западный административный округ, "
u8"locality: Москва, suburb: Раменки"),
());
TEST(HasName(regions, u8"Россия, region: Москва, subregion: Западный административный округ, "
u8"locality: Москва, suburb: Раменки, sublocality: Воробъёвы горы"),
());
*/
TEST(HasName(regions, u8"Россия, region: Москва, subregion: Западный административный округ, "
u8"locality: Москва, sublocality: Центр"),
());
TEST(!ContainsSubname(regions, u8"Тропарёво"), ());
}
UNIT_TEST(RegionsBuilderTest_GenerateRusSPetersburgSuburb)
{
TagValue const admin{"admin_level"};
TagValue const place{"place"};
TagValue const name{"name"};
TagValue const ba{"boundary", "administrative"};
auto regions = GenerateTestRegions({
{1, {name = u8"Россия", admin = "2", ba}, {{0, 0}, {50, 50}}},
{2, {name = u8"Северо-Западный федеральный округ", admin = "3", ba}, {{10, 10}, {20, 20}}},
{3, {name = u8"Санкт-Петербург", admin = "4", ba}, {{12, 12}, {18, 18}}},
{4, {name = u8"Санкт-Петербург", place = "city"}, {{12, 12}, {17, 17}}},
{5, {name = u8"Центральный район", admin = "5", ba}, {{14, 14}, {16, 16}}},
{6, {name = u8"Дворцовый округ", admin = "8", ba}, {{14, 14}, {15, 15}}},
});
TEST(HasName(regions, u8"Россия, region: Санкт-Петербург, locality: Санкт-Петербург"), ());
/* FIXME:
TEST(HasName(regions, u8"Россия, region: Санкт-Петербург, locality: Санкт-Петербург, "
u8"suburb: Центральный район"),
());
TEST(HasName(regions, u8"Россия, region: Санкт-Петербург, locality: Санкт-Петербург, "
u8"suburb: Центральный район, sublocality: Дворцовый округ"),
());
*/
}
// Transliteration tests ---------------------------------------------------------------------------
using Translations = std::vector<std::pair<std::string, std::string>>;
bool TestTransliteration(Translations const & translations,
std::string const & expectedTransliteration)

View file

@ -69,7 +69,7 @@ char const * GetLabel(PlaceLevel level)
CollectorRegionInfo::CollectorRegionInfo(std::string const & filename) : m_filename(filename) {}
void CollectorRegionInfo::CollectFeature(const FeatureBuilder &, OsmElement const & el)
void CollectorRegionInfo::Collect(OsmElement const & el)
{
base::GeoObjectId const osmId = GetGeoObjectId(el);
RegionData regionData;

View file

@ -118,7 +118,7 @@ public:
CollectorRegionInfo(std::string const & filename);
// CollectorInterface overrides:
void CollectFeature(feature::FeatureBuilder const &, OsmElement const & el) override;
void Collect(OsmElement const & el) override;
void Save() override;
private:

View file

@ -4,6 +4,9 @@ namespace generator
{
namespace regions
{
void CountrySpecifier::AdjustRegionsLevel(Node::PtrList & outers)
{ }
PlaceLevel CountrySpecifier::GetLevel(Region const & region) const
{
auto const placeType = region.GetPlaceType();

View file

@ -2,6 +2,7 @@
#include "generator/regions/collector_region_info.hpp"
#include "generator/regions/level_region.hpp"
#include "generator/regions/node.hpp"
#include "generator/regions/region.hpp"
namespace generator
@ -13,6 +14,8 @@ class CountrySpecifier
public:
virtual ~CountrySpecifier() = default;
virtual void AdjustRegionsLevel(Node::PtrList & outers);
virtual PlaceLevel GetLevel(Region const & region) const;
// Return -1 - |l| is under place of |r|, 0 - undefined relation, 1 - |r| is under place of |l|.
// Non-transitive.

View file

@ -1,5 +1,7 @@
#include "generator/regions/regions_builder.hpp"
#include "generator/regions/specs/rus.hpp"
#include "base/assert.hpp"
#include "base/thread_pool_computational.hpp"
#include "base/stl_helpers.hpp"
@ -181,6 +183,8 @@ void RegionsBuilder::ForEachCountry(CountryFn fn)
std::copy_if(std::begin(countries), std::end(countries), std::back_inserter(outers), pred);
auto countryTrees = BuildCountryRegionTrees(outers, *countrySpecifier);
countrySpecifier->AdjustRegionsLevel(countryTrees);
fn(countryName, countryTrees);
}
}
@ -208,6 +212,9 @@ Node::PtrList RegionsBuilder::BuildCountryRegionTrees(Regions const & outers,
std::unique_ptr<CountrySpecifier> RegionsBuilder::GetCountrySpecifier(std::string const & countryName)
{
if (countryName == u8"Россия" || countryName == u8"Российская Федерация" || countryName == u8"РФ")
return std::make_unique<specs::RusSpecifier>();
return std::make_unique<CountrySpecifier>();
}
} // namespace regions

View file

@ -0,0 +1,150 @@
#include "generator/regions/specs/rus.hpp"
#include "base/stl_helpers.hpp"
namespace generator
{
namespace regions
{
namespace specs
{
void RusSpecifier::AdjustRegionsLevel(Node::PtrList & outers)
{
AdjustMoscowAdministrativeDivisions(outers);
}
void RusSpecifier::AdjustMoscowAdministrativeDivisions(Node::PtrList & outers)
{
for (auto const & tree : outers)
AdjustMoscowAdministrativeDivisions(tree);
if (!m_moscowRegionWasProcessed)
LOG(LWARNING, ("Failed to find Moscow region"));
if (!m_moscowCityWasProcessed)
LOG(LWARNING, ("Failed to find Moscow city"));
}
void RusSpecifier::AdjustMoscowAdministrativeDivisions(Node::Ptr const & tree)
{
auto const & region = tree->GetData();
if (region.GetAdminLevel() == AdminLevel::Four && region.GetName() == u8"Москва")
{
for (auto & subtree : tree->GetChildren())
{
MarkMoscowSubregionsByAdministrativeOkrugs(subtree);
AdjustMoscowCitySuburbs(subtree);
}
return;
}
if (region.GetAdminLevel() > AdminLevel::Four)
return;
for (auto & subtree : tree->GetChildren())
AdjustMoscowAdministrativeDivisions(subtree);
}
void RusSpecifier::MarkMoscowSubregionsByAdministrativeOkrugs(Node::Ptr & tree)
{
auto & region = tree->GetData();
auto const adminLevel = region.GetAdminLevel();
if (adminLevel == AdminLevel::Five)
{
region.SetLevel(PlaceLevel::Subregion);
m_moscowRegionWasProcessed = true;
return;
}
if (adminLevel > AdminLevel::Five)
return;
for (auto & subtree : tree->GetChildren())
MarkMoscowSubregionsByAdministrativeOkrugs(subtree);
}
void RusSpecifier::AdjustMoscowCitySuburbs(Node::Ptr const & tree)
{
auto & region = tree->GetData();
if (region.GetPlaceType() == PlaceType::City && region.GetName() == u8"Москва")
{
for (auto & subtree : tree->GetChildren())
MarkMoscowSuburbsByAdministrativeDistrics(subtree);
return;
}
if (region.GetLevel() >= PlaceLevel::Locality)
return;
for (auto & subtree : tree->GetChildren())
AdjustMoscowCitySuburbs(subtree);
}
void RusSpecifier::MarkMoscowSuburbsByAdministrativeDistrics(Node::Ptr & tree)
{
auto & region = tree->GetData();
if (AdminLevel::Eight == region.GetAdminLevel())
{
MarkMoscowAdministrativeDistric(tree);
return;
}
if (PlaceLevel::Suburb == region.GetLevel())
region.SetLevel(PlaceLevel::Sublocality);
for (auto & subtree : tree->GetChildren())
MarkMoscowSuburbsByAdministrativeDistrics(subtree);
}
void RusSpecifier::MarkMoscowAdministrativeDistric(Node::Ptr & node)
{
auto & region = node->GetData();
region.SetLevel(PlaceLevel::Suburb);
m_moscowCityWasProcessed = true;
for (auto & subtree : node->GetChildren())
MarkAllSuburbsToSublocalities(subtree);
}
void RusSpecifier::MarkAllSuburbsToSublocalities(Node::Ptr & tree)
{
auto & region = tree->GetData();
auto const level = region.GetLevel();
if (level == PlaceLevel::Locality) // nested locality
return;
if (level == PlaceLevel::Suburb)
region.SetLevel(PlaceLevel::Sublocality);
for (auto & subtree : tree->GetChildren())
MarkAllSuburbsToSublocalities(subtree);
}
PlaceLevel RusSpecifier::GetLevel(Region const & region) const
{
auto placeLevel = CountrySpecifier::GetLevel(region.GetPlaceType());
if (placeLevel != PlaceLevel::Unknown)
return placeLevel;
return GetRussiaPlaceLevel(region.GetAdminLevel());
}
// static
PlaceLevel RusSpecifier::GetRussiaPlaceLevel(AdminLevel adminLevel)
{
switch (adminLevel)
{
case AdminLevel::Two:
return PlaceLevel::Country;
case AdminLevel::Four:
return PlaceLevel::Region;
case AdminLevel::Six:
return PlaceLevel::Subregion;
default:
break;
}
return PlaceLevel::Unknown;
}
} // namespace specs
} // namespace regions
} // namespace generator

View file

@ -0,0 +1,38 @@
#pragma once
#include "generator/place_node.hpp"
#include "generator/regions/collector_region_info.hpp"
#include "generator/regions/country_specifier.hpp"
#include "generator/regions/region.hpp"
namespace generator
{
namespace regions
{
namespace specs
{
class RusSpecifier final : public CountrySpecifier
{
public:
// CountrySpecifier overrides:
void AdjustRegionsLevel(Node::PtrList & outers) override;
PlaceLevel GetLevel(Region const & region) const override;
private:
void AdjustMoscowAdministrativeDivisions(Node::PtrList & outers);
void AdjustMoscowAdministrativeDivisions(Node::Ptr const & tree);
void MarkMoscowSubregionsByAdministrativeOkrugs(Node::Ptr & node);
void AdjustMoscowCitySuburbs(Node::Ptr const & tree);
void MarkMoscowSuburbsByAdministrativeDistrics(Node::Ptr & tree);
void MarkMoscowAdministrativeDistric(Node::Ptr & node);
void MarkAllSuburbsToSublocalities(Node::Ptr & tree);
static PlaceLevel GetRussiaPlaceLevel(AdminLevel adminLevel);
bool m_moscowRegionWasProcessed{false};
bool m_moscowCityWasProcessed{false};
};
} // namespace specs
} // namespace regions
} // namespace generator