diff --git a/3party/jansson/myjansson.cpp b/3party/jansson/myjansson.cpp index 10b36457df..e6a2ebb4cb 100644 --- a/3party/jansson/myjansson.cpp +++ b/3party/jansson/myjansson.cpp @@ -18,6 +18,11 @@ string FromJSONToString(json_t * root) namespace base { json_t * GetJSONObligatoryField(json_t * root, std::string const & field) +{ + return GetJSONObligatoryField(root, field.c_str()); +} + +json_t * GetJSONObligatoryField(json_t * root, char const * field) { auto * value = base::GetJSONOptionalField(root, field); if (!value) @@ -26,10 +31,15 @@ json_t * GetJSONObligatoryField(json_t * root, std::string const & field) } json_t * GetJSONOptionalField(json_t * root, std::string const & field) +{ + return GetJSONOptionalField(root, field.c_str()); +} + +json_t * GetJSONOptionalField(json_t * root, char const * field) { if (!json_is_object(root)) MYTHROW(base::Json::Exception, ("Bad json object while parsing", field)); - return json_object_get(root, field.c_str()); + return json_object_get(root, field); } bool JSONIsNull(json_t * root) { return json_is_null(root); } diff --git a/3party/jansson/myjansson.hpp b/3party/jansson/myjansson.hpp index 31e55bc369..a34daaf6e0 100644 --- a/3party/jansson/myjansson.hpp +++ b/3party/jansson/myjansson.hpp @@ -38,6 +38,7 @@ public: Json() = default; explicit Json(std::string_view const & s) { ParseFrom(s); } + explicit Json(JSONPtr const & json) : m_handle{json.get()} { } Json GetDeepCopy() const { @@ -60,7 +61,9 @@ private: JsonHandle m_handle; }; +json_t * GetJSONObligatoryField(json_t * root, char const * field); json_t * GetJSONObligatoryField(json_t * root, std::string const & field); +json_t * GetJSONOptionalField(json_t * root, char const * field); json_t * GetJSONOptionalField(json_t * root, std::string const & field); bool JSONIsNull(json_t * root); } // namespace base diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index 1592d7589b..aa3f7a920e 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -98,6 +98,8 @@ set(SRC geo_objects/key_value_storage.hpp geo_objects/region_info_getter.cpp geo_objects/region_info_getter.hpp + geo_objects/streets_builder.cpp + geo_objects/streets_builder.hpp holes.cpp holes.hpp intermediate_data.cpp diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp index 728d3ae0e2..162d104f6b 100644 --- a/generator/geo_objects/geo_objects.cpp +++ b/generator/geo_objects/geo_objects.cpp @@ -1,12 +1,10 @@ #include "generator/geo_objects/geo_objects.hpp" +#include "generator/feature_builder.hpp" #include "generator/geo_objects/geo_object_info_getter.hpp" #include "generator/geo_objects/key_value_storage.hpp" #include "generator/geo_objects/region_info_getter.hpp" - -#include "generator/geo_objects/key_value_storage.hpp" - -#include "generator/feature_builder.hpp" +#include "generator/geo_objects/streets_builder.hpp" #include "generator/locality_sorter.hpp" #include "generator/regions/region_base.hpp" @@ -189,7 +187,7 @@ void BuildGeoObjectsWithoutAddresses(GeoObjectInfoGetter const & geoObjectInfoGe { size_t countGeoObjects = 0; auto const fn = [&](FeatureBuilder1 & fb, uint64_t /* currPos */) { - if (IsBuilding(fb) || HasHouse(fb)) + if (IsBuilding(fb) || HasHouse(fb) || StreetsBuilder::IsStreet(fb)) return; auto const house = FindHousePoi(fb, geoObjectInfoGetter); @@ -229,8 +227,11 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex, RegionInfoGetter regionInfoGetter{pathInRegionsIndex, pathInRegionsKv}; LOG(LINFO, ("Size of regions key-value storage:", regionInfoGetter.GetStorage().Size())); - std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress); + StreetsBuilder streetsBuilder{regionInfoGetter}; std::ofstream streamGeoObjectsKv(pathOutGeoObjectsKv); + streetsBuilder.Build(pathInGeoObjectsTmpMwm, streamGeoObjectsKv); + LOG(LINFO, ("Streets was built.")); + BuildGeoObjectsWithAddresses(regionInfoGetter, pathInGeoObjectsTmpMwm, streamGeoObjectsKv, verbose); LOG(LINFO, ("Geo objects with addresses were built.")); @@ -244,6 +245,7 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex, return false; GeoObjectInfoGetter geoObjectInfoGetter{std::move(*geoObjectIndex), std::move(geoObjectsKv)}; + std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress); BuildGeoObjectsWithoutAddresses(geoObjectInfoGetter, pathInGeoObjectsTmpMwm, streamGeoObjectsKv, streamIdsWithoutAddress, verbose); LOG(LINFO, ("Geo objects without addresses were built.")); diff --git a/generator/geo_objects/region_info_getter.cpp b/generator/geo_objects/region_info_getter.cpp index 1b138546aa..39650551d8 100644 --- a/generator/geo_objects/region_info_getter.cpp +++ b/generator/geo_objects/region_info_getter.cpp @@ -14,9 +14,15 @@ RegionInfoGetter::RegionInfoGetter(std::string const & indexPath, std::string co { } boost::optional RegionInfoGetter::FindDeepest(m2::PointD const & point) const +{ + return FindDeepest(point, [] (...) { return true; }); +} + +boost::optional RegionInfoGetter::FindDeepest( + m2::PointD const & point, Selector const & selector) const { auto const ids = SearchObjectsInIndex(point); - return GetDeepest(ids); + return GetDeepest(ids, selector); } std::vector RegionInfoGetter::SearchObjectsInIndex(m2::PointD const & point) const @@ -27,7 +33,8 @@ std::vector RegionInfoGetter::SearchObjectsInIndex(m2::PointD return ids; } -boost::optional RegionInfoGetter::GetDeepest(std::vector const & ids) const +boost::optional RegionInfoGetter::GetDeepest( + std::vector const & ids, Selector const & selector) const { boost::optional deepest; int deepestRank = 0; @@ -49,7 +56,7 @@ boost::optional RegionInfoGetter::GetDeepest(std::vector(id.GetEncodedId()), temp); diff --git a/generator/geo_objects/region_info_getter.hpp b/generator/geo_objects/region_info_getter.hpp index 3d422e616c..3c0d4d9a56 100644 --- a/generator/geo_objects/region_info_getter.hpp +++ b/generator/geo_objects/region_info_getter.hpp @@ -24,16 +24,20 @@ namespace geo_objects class RegionInfoGetter { public: + using Selector = std::function; + RegionInfoGetter(std::string const & indexPath, std::string const & kvPath); boost::optional FindDeepest(m2::PointD const & point) const; + boost::optional FindDeepest(m2::PointD const & point, Selector const & selector) const; KeyValueStorage const & GetStorage() const noexcept; private: using IndexReader = ReaderPtr; std::vector SearchObjectsInIndex(m2::PointD const & point) const; - boost::optional GetDeepest(std::vector const & ids) const; + boost::optional GetDeepest(std::vector const & ids, + Selector const & selector) const; int GetRank(base::Json const & json) const; indexer::RegionsIndex m_index; diff --git a/generator/geo_objects/streets_builder.cpp b/generator/geo_objects/streets_builder.cpp new file mode 100644 index 0000000000..10477a657d --- /dev/null +++ b/generator/geo_objects/streets_builder.cpp @@ -0,0 +1,139 @@ +#include "generator/geo_objects/streets_builder.hpp" + +#include "indexer/classificator.hpp" + +#include "base/logging.hpp" + +#include + +#include "3party/jansson/myjansson.hpp" + +namespace generator +{ +namespace geo_objects +{ +StreetsBuilder::StreetsBuilder(RegionInfoGetter const & regionInfoGetter) + : m_regionInfoGetter{regionInfoGetter} +{ } + +void StreetsBuilder::Build(std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamGeoObjectsKv) +{ + auto const transform = [this, &streamGeoObjectsKv](FeatureBuilder1 & fb, uint64_t /* currPos */) { + if (!IsStreet(fb)) + return; + + auto const region = FindStreetRegionOwner(fb); + if (!region) + return; + + if (!InsertStreet(*region, fb)) + return; + + auto const value = MakeStreetValue(fb, *region); + auto const id = static_cast(fb.GetMostGenericOsmId().GetEncodedId()); + streamGeoObjectsKv << id << " " << value.get() << "\n"; + }; + + feature::ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, transform); +} + +boost::optional StreetsBuilder::FindStreetRegionOwner(FeatureBuilder1 & fb) +{ + auto const & line = fb.GetOuterGeometry(); + auto const & startPoint = line.front(); + auto const & owner = FindStreetRegionOwner(startPoint); + if (!owner) + return {}; + + auto const & finishPoint = line.back(); + if (startPoint != finishPoint) + { + auto const & finishPointOwner = FindStreetRegionOwner(finishPoint); + if (!finishPointOwner || finishPointOwner->first != owner->first) + LOG(LDEBUG, ("Street", fb.GetMostGenericOsmId(), fb.GetName(), "is in several regions")); + } + + return owner; +} + +boost::optional StreetsBuilder::FindStreetRegionOwner(m2::PointD const & point) +{ + auto const isStreetAdministrator = [] (base::Json const & region) { + auto const && properties = base::GetJSONObligatoryField(region.get(), "properties"); + auto const && address = base::GetJSONObligatoryField(properties, "address"); + + if (base::GetJSONOptionalField(address, "suburb")) + return false; + if (base::GetJSONOptionalField(address, "sublocality")) + return false; + + return true; + }; + + return m_regionInfoGetter.FindDeepest(point, isStreetAdministrator); +} + +bool StreetsBuilder::InsertStreet(KeyValue const & region, FeatureBuilder1 & fb) +{ + auto & regionStreets = m_regionsStreets[region.first]; + auto emplace = regionStreets.emplace(fb.GetName()); + return emplace.second; +} + +std::unique_ptr StreetsBuilder::MakeStreetValue( + FeatureBuilder1 const & fb, KeyValue const & region) +{ + auto const && regionProperties = base::GetJSONObligatoryField(region.second.get(), "properties"); + auto const && regionAddress = base::GetJSONObligatoryField(regionProperties, "address"); + auto address = base::JSONPtr{json_deep_copy(regionAddress)}; + ToJSONObject(*address, "street", fb.GetName()); + + auto properties = base::NewJSONObject(); + ToJSONObject(*properties, "address", std::move(address)); + ToJSONObject(*properties, "name", fb.GetName()); + ToJSONObject(*properties, "pid", region.first); + + auto streetObject = base::NewJSONObject(); + ToJSONObject(*streetObject, "properties", std::move(properties)); + + auto const value = json_dumps(streetObject.get(), JSON_COMPACT); + return std::unique_ptr{value}; +} + +// static +bool StreetsBuilder::IsStreet(OsmElement const & element) +{ + if (!element.IsWay() && !element.IsRelation()) + return false; + + auto const & tags = element.Tags(); + + auto const isHighway = std::any_of(std::cbegin(tags), std::cend(tags), [] (auto const & tag) { + return tag.key == "highway"; + }); + if (!isHighway) + return false; + + auto const hasName = std::any_of(std::cbegin(tags), std::cend(tags), [] (auto const & tag) { + return tag.key == "name"; + }); + if (!hasName) + return false; + + return true; +} + +// static +bool StreetsBuilder::IsStreet(FeatureBuilder1 const & fb) +{ + if (!fb.IsLine() && !fb.IsArea()) + return false; + + static auto const highwayType = classif().GetTypeByPath({"highway"}); + if (fb.FindType(highwayType, 1) == ftype::GetEmptyValue()) + return false; + + return !fb.GetName().empty(); +} +} // namespace geo_objects +} // namespace generator diff --git a/generator/geo_objects/streets_builder.hpp b/generator/geo_objects/streets_builder.hpp new file mode 100644 index 0000000000..5de7aadc4c --- /dev/null +++ b/generator/geo_objects/streets_builder.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "generator/feature_builder.hpp" +#include "generator/geo_objects/key_value_storage.hpp" +#include "generator/geo_objects/region_info_getter.hpp" +#include "generator/osm_element.hpp" + +#include "coding/reader.hpp" + +#include "geometry/point2d.hpp" + +#include +#include +#include +#include + +#include + +namespace generator +{ +namespace geo_objects +{ +class StreetsBuilder +{ +public: + StreetsBuilder(RegionInfoGetter const & regionInfoGetter); + + void Build(std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamGeoObjectsKv); + + static bool IsStreet(OsmElement const & element); + static bool IsStreet(FeatureBuilder1 const & fb); + +private: + boost::optional FindStreetRegionOwner(FeatureBuilder1 & fb); + boost::optional FindStreetRegionOwner(m2::PointD const & point); + bool InsertStreet(KeyValue const & region, FeatureBuilder1 & fb); + std::unique_ptr MakeStreetValue(FeatureBuilder1 const & fb, KeyValue const & region); + + std::unordered_map> m_regionsStreets; + RegionInfoGetter const & m_regionInfoGetter; +}; +} // namespace geo_objects +} // namespace generator diff --git a/generator/osm_element.cpp b/generator/osm_element.cpp index d44fbe6e17..fddf9a8adf 100644 --- a/generator/osm_element.cpp +++ b/generator/osm_element.cpp @@ -32,7 +32,7 @@ std::string DebugPrint(OsmElement::EntityType e) } -void OsmElement::AddTag(std::string const & k, std::string const & v) +void OsmElement::AddTag(std::string_view const & k, std::string_view const & v) { // Seems like source osm data has empty values. They are useless for us. if (k.empty() || v.empty()) @@ -65,12 +65,12 @@ void OsmElement::AddTag(std::string const & k, std::string const & v) SKIP_KEY("official_name"); #undef SKIP_KEY - std::string value = v; + std::string value{std::string{v}}; strings::Trim(value); - m_tags.emplace_back(k, value); + m_tags.emplace_back(std::string{k}, std::move(value)); } -bool OsmElement::HasTag(std::string const & k, std::string const & v) const +bool OsmElement::HasTag(std::string_view const & k, std::string_view const & v) const { return std::any_of(m_tags.begin(), m_tags.end(), [&](auto const & t) { return t.key == k && t.value == v; diff --git a/generator/osm_element.hpp b/generator/osm_element.hpp index f4ae77f646..bf9406a5d0 100644 --- a/generator/osm_element.hpp +++ b/generator/osm_element.hpp @@ -5,6 +5,8 @@ #include "base/math.hpp" #include "base/string_utils.hpp" +#include "std/string_view.hpp" + #include #include #include @@ -139,8 +141,8 @@ struct OsmElement m_members.emplace_back(ref, type, role); } - void AddTag(std::string const & k, std::string const & v); - bool HasTag(std::string const & k, std::string const & v) const; + void AddTag(std::string_view const & k, std::string_view const & v); + bool HasTag(std::string_view const & k, std::string_view const & v) const; bool HasAnyTag(std::unordered_multimap const & tags) const; template diff --git a/generator/translator_geo_objects.cpp b/generator/translator_geo_objects.cpp index b7325c32a4..71df787c4b 100644 --- a/generator/translator_geo_objects.cpp +++ b/generator/translator_geo_objects.cpp @@ -2,6 +2,7 @@ #include "generator/feature_maker.hpp" #include "generator/filter_interface.hpp" +#include "generator/geo_objects/streets_builder.hpp" #include "generator/intermediate_data.hpp" #include "generator/osm_element.hpp" #include "generator/osm_element_helpers.hpp" @@ -16,12 +17,13 @@ public: // FilterInterface overrides: bool IsAccepted(OsmElement const & element) override { - return osm_element::IsBuilding(element) || osm_element::IsPoi(element); + return osm_element::IsBuilding(element) || osm_element::IsPoi(element) || + geo_objects::StreetsBuilder::IsStreet(element); } bool IsAccepted(FeatureBuilder1 const & feature) override { - return feature.GetParams().IsValid() && !feature.IsLine(); + return feature.GetParams().IsValid(); } }; } // namespace