diff --git a/generator/feature_builder.cpp b/generator/feature_builder.cpp index dda6b2e..ddb5354 100644 --- a/generator/feature_builder.cpp +++ b/generator/feature_builder.cpp @@ -649,7 +649,16 @@ void FeatureBuilder::SerializeLocalityObject(serial::GeometryCodingParams const return; } - CHECK_EQUAL(type, GeomType::Area, ("Supported types are Point and Area")); + if (type == GeomType::Line) + { + uint32_t const ptsCount = base::asserted_cast(data.m_innerPts.size()); + CHECK_GREATER(ptsCount, 1, ()); + WriteToSink(sink, ptsCount); + serial::SaveInnerPath(data.m_innerPts, params, sink); + return; + } + + CHECK_EQUAL(type, GeomType::Area, ()); uint32_t trgCount = base::asserted_cast(data.m_innerTrg.size()); CHECK_GREATER(trgCount, 2, ()); diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 180dba3..f912ea8 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -317,6 +317,7 @@ int GeneratorToolMain(int argc, char ** argv) if (options.m_generate_geo_objects_index) { if (!feature::GenerateGeoObjectsData(options.m_geo_objects_features, + options.m_streets_features, options.m_nodes_list_path, locDataFile)) { LOG(LCRITICAL, ("Error generating geo objects data.")); diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp index 7d82bd0..07f3f14 100644 --- a/generator/geo_objects/geo_objects.cpp +++ b/generator/geo_objects/geo_objects.cpp @@ -212,7 +212,8 @@ boost::optional> MakeTempGeoObjectsIndex( { auto const dataFile = GetPlatform().TmpPathForFile(); SCOPE_GUARD(removeDataFile, std::bind(Platform::RemoveFileIfExists, std::cref(dataFile))); - if (!GenerateGeoObjectsData(pathToGeoObjectsTmpMwm, "" /* nodesFile */, dataFile)) + if (!GenerateGeoObjectsData(pathToGeoObjectsTmpMwm, "" /* streetFeaturesFile */, + "" /* nodesFile */, dataFile)) { LOG(LCRITICAL, ("Error generating geo objects data.")); return {}; diff --git a/generator/locality_sorter.cpp b/generator/locality_sorter.cpp index 67bab43..570fcc4 100644 --- a/generator/locality_sorter.cpp +++ b/generator/locality_sorter.cpp @@ -2,6 +2,7 @@ #include "generator/geo_objects/geo_objects_filter.hpp" #include "generator/geometry_holder.hpp" +#include "generator/streets/streets_filter.hpp" #include "generator/utils.hpp" #include "indexer/data_header.hpp" @@ -113,6 +114,9 @@ public: SimplifyPoints(distFn, scales::GetUpperScale(), holder.GetSourcePoints(), points); + if (fb.IsLine()) + holder.AddPoints(points, 0); + // For areas we save outer geometry only. if (fb.IsArea() && holder.NeedProcessTriangles()) { @@ -235,22 +239,39 @@ bool GenerateLocalityDataImpl(FeaturesCollector & collector, namespace feature { -bool GenerateGeoObjectsData(string const & featuresFile, string const & nodesFile, - string const & dataFile) +bool GenerateGeoObjectsData(string const & geoObjectsFeaturesFile, + string const & streetFeaturesFile, + string const & nodesFile, string const & dataFile) { + auto featuresFile = geoObjectsFeaturesFile; + auto const geoObjectsAndStreetsFeaturesFile = GetPlatform().TmpPathForFile(); + SCOPE_GUARD(geoObjectsAndStreetsFeaturesFileGuard, + std::bind(Platform::RemoveFileIfExists, geoObjectsAndStreetsFeaturesFile)); + if (!streetFeaturesFile.empty()) + { + auto features = std::ofstream{geoObjectsAndStreetsFeaturesFile, std::ios_base::binary}; + for (auto const & file : {geoObjectsFeaturesFile , streetFeaturesFile}) + { + auto fileStream = std::ifstream{file, std::ios_base::binary}; + features << fileStream.rdbuf(); + } + featuresFile = geoObjectsAndStreetsFeaturesFile; + } + set nodeIds; if (!ParseNodes(nodesFile, nodeIds)) return false; auto const needSerialize = [&nodeIds](FeatureBuilder & fb) { - if (!fb.IsPoint() && !fb.IsArea()) - return false; - using generator::geo_objects::GeoObjectsFilter; + using generator::streets::StreetsFilter; if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb)) return true; + if (StreetsFilter::IsStreet(fb)) + return true; + if (GeoObjectsFilter::IsPoi(fb)) return 0 != nodeIds.count(fb.GetMostGenericOsmId().GetEncodedId()); diff --git a/generator/locality_sorter.hpp b/generator/locality_sorter.hpp index 723d2b2..e983967 100644 --- a/generator/locality_sorter.hpp +++ b/generator/locality_sorter.hpp @@ -8,7 +8,9 @@ namespace feature // @param featuresDir - path to folder with pregenerated features data; // @param nodesFile - path to file with list of node ids we need to add to output; // @param out - output file name; -bool GenerateGeoObjectsData(std::string const & featuresFile, std::string const & nodesFile, +bool GenerateGeoObjectsData(std::string const & geoObjectsFeaturesFile, + std::string const & streetFeaturesFile, + std::string const & nodesFile, std::string const & out); // Generates data for RegionsIndexBuilder from input feature-dat-files. diff --git a/generator/streets/street_geometry.cpp b/generator/streets/street_geometry.cpp index 91cfcf1..2855992 100644 --- a/generator/streets/street_geometry.cpp +++ b/generator/streets/street_geometry.cpp @@ -51,6 +51,16 @@ void StreetGeometry::SetPin(Pin && pin) m_pin = std::move(pin); } +boost::optional const & StreetGeometry::GetPin() const +{ + return m_pin; +} + +HighwayGeometry const * StreetGeometry::GetHighwayGeometry() const +{ + return m_highwayGeometry.get(); +} + void StreetGeometry::AddHighwayLine(base::GeoObjectId const & osmId, std::vector const & line) { if (!m_highwayGeometry) @@ -85,6 +95,21 @@ Pin HighwayGeometry::ChoosePin() const return ChooseMultilinePin(); } +m2::RectD const & HighwayGeometry::GetBbox() const +{ + return m_limitRect; +} + +std::vector const & HighwayGeometry::GetAreaParts() const +{ + return m_areaParts; +} + +HighwayGeometry::MultiLine const & HighwayGeometry::GetMultiLine() const +{ + return m_multiLine; +} + Pin HighwayGeometry::ChooseMultilinePin() const { auto const & lines = m_multiLine.m_lines; @@ -145,11 +170,6 @@ void HighwayGeometry::AddArea(base::GeoObjectId const & osmId, std::vector const & points) { feature::CalcRect(points, m_limitRect); @@ -315,13 +335,14 @@ double HighwayGeometry::LineSegment::CalculateLength() const noexcept // HighwayGeometry::AreaPart --------------------------------------------------------------------------------- -HighwayGeometry::AreaPart::AreaPart(base::GeoObjectId const & osmId, std::vector const & polygon) - : m_osmId{osmId} +HighwayGeometry::AreaPart::AreaPart(base::GeoObjectId const & osmId, + std::vector const & border) + : m_osmId{osmId}, m_border{border} { - CHECK_GREATER_OR_EQUAL(polygon.size(), 3, ()); + CHECK_GREATER_OR_EQUAL(border.size(), 3, ()); auto boostPolygon = boost_helpers::BoostPolygon{}; - for (auto const & p : polygon) + for (auto const & p : border) boost::geometry::append(boostPolygon, boost_helpers::BoostPoint{p.x, p.y}); boost::geometry::correct(boostPolygon); diff --git a/generator/streets/street_geometry.hpp b/generator/streets/street_geometry.hpp index 3b20397..d031c83 100644 --- a/generator/streets/street_geometry.hpp +++ b/generator/streets/street_geometry.hpp @@ -32,6 +32,8 @@ public: Pin GetOrChoosePin() const; // Bbox may be the minimum bounding box with feature type margin. m2::RectD GetBbox() const; + boost::optional const & GetPin() const; + HighwayGeometry const * GetHighwayGeometry() const; void SetPin(Pin && pin); void AddHighwayLine(base::GeoObjectId const & osmId, std::vector const & line); @@ -49,13 +51,6 @@ private: class HighwayGeometry { public: - Pin ChoosePin() const; - m2::RectD const & GetBbox() const; - - void AddLine(base::GeoObjectId const & osmId, std::vector const & line); - void AddArea(base::GeoObjectId const & osmId, std::vector const & border); - -private: struct LineSegment { LineSegment(base::GeoObjectId const & osmId, std::vector const & points); @@ -93,13 +88,23 @@ private: struct AreaPart { - AreaPart(base::GeoObjectId const & osmId, std::vector const & polygon); + AreaPart(base::GeoObjectId const & osmId, std::vector const & border); base::GeoObjectId m_osmId; + std::vector m_border; m2::PointD m_center; double m_area; }; + Pin ChoosePin() const; + m2::RectD const & GetBbox() const; + std::vector const & GetAreaParts() const; + MultiLine const & GetMultiLine() const; + + void AddLine(base::GeoObjectId const & osmId, std::vector const & line); + void AddArea(base::GeoObjectId const & osmId, std::vector const & border); + +private: Pin ChooseMultilinePin() const; Pin ChooseLinePin(Line const & line, double disposeDistance) const; Pin ChooseAreaPin() const; diff --git a/generator/streets/streets.cpp b/generator/streets/streets.cpp index 6f22e95..a833597 100644 --- a/generator/streets/streets.cpp +++ b/generator/streets/streets.cpp @@ -36,6 +36,9 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & streetsBuilder.AssembleBindings(pathInGeoObjectsTmpMwm); LOG(LINFO, ("Binding's streets were built.")); + streetsBuilder.RegenerateAggreatedStreetsFeatures(pathInStreetsTmpMwm); + LOG(LINFO, ("Streets features are aggreated into", pathInStreetsTmpMwm)); + std::ofstream streamStreetsKv(pathOutStreetsKv); streetsBuilder.SaveStreetsKv(streamStreetsKv); LOG(LINFO, ("Streets key-value storage saved to", pathOutStreetsKv)); diff --git a/generator/streets/streets_builder.cpp b/generator/streets/streets_builder.cpp index 20addf8..77102fd 100644 --- a/generator/streets/streets_builder.cpp +++ b/generator/streets/streets_builder.cpp @@ -1,14 +1,20 @@ #include "generator/streets/streets_builder.hpp" + #include "generator/key_value_storage.hpp" #include "generator/streets/street_regions_tracing.hpp" #include "generator/translation.hpp" +#include "coding/internal/file_data.hpp" + #include "indexer/classificator.hpp" #include "indexer/ftypes_matcher.hpp" #include "geometry/mercator.hpp" +#include "platform/platform.hpp" + #include "base/logging.hpp" +#include "base/scope_guard.hpp" #include @@ -47,6 +53,72 @@ void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm ForEachParallelFromDatRawFormat(m_threadsCount, pathInGeoObjectsTmpMwm, transform); } +void StreetsBuilder::RegenerateAggreatedStreetsFeatures( + std::string const & pathStreetsTmpMwm) +{ + auto const aggregatedStreetsTmpFile = GetPlatform().TmpPathForFile(); + SCOPE_GUARD(aggregatedStreetsTmpFileGuard, + std::bind(Platform::RemoveFileIfExists, aggregatedStreetsTmpFile)); + FeaturesCollector collector(aggregatedStreetsTmpFile); + + std::set processedStreets; + auto const transform = [&](FeatureBuilder & fb, uint64_t /* currPos */) { + auto street = m_streetFeatures2Streets.find(fb.GetMostGenericOsmId()); + if (street == m_streetFeatures2Streets.end()) + return; + + if (!processedStreets.insert(street->second).second) + return; + + WriteAsAggregatedStreet(fb, *street->second, collector); + }; + ForEachFromDatRawFormat(pathStreetsTmpMwm, transform); + + CHECK(base::RenameFileX(aggregatedStreetsTmpFile, pathStreetsTmpMwm), ()); +} + +void StreetsBuilder::WriteAsAggregatedStreet(FeatureBuilder & fb, Street const & street, + FeaturesCollector & collector) const +{ + fb.GetParams().name = street.m_name; + + auto const & geometry = street.m_geometry; + auto const & pin = geometry.GetOrChoosePin(); + fb.SetOsmId(pin.m_osmId); + + if (auto const & pin = geometry.GetPin()) + { + fb.ResetGeometry(); + fb.SetCenter(pin->m_position); + collector.Collect(fb); + } + + auto const * highwayGeometry = geometry.GetHighwayGeometry(); + if (!highwayGeometry) + return; + + for (auto const & area : highwayGeometry->GetAreaParts()) + { + fb.ResetGeometry(); + fb.GetParams().SetGeomType(feature::GeomType::Area); + auto polygon = area.m_border; + fb.AddPolygon(polygon); + collector.Collect(fb); + } + + for (auto const & line : highwayGeometry->GetMultiLine().m_lines) + { + for (auto const & segment : line.m_segments) + { + fb.ResetGeometry(); + fb.SetLinear(); + for (auto const & point : segment.m_points) + fb.AddPoint(point); + collector.Collect(fb); + } + } +} + void StreetsBuilder::SaveStreetsKv(std::ostream & streamStreetsKv) { for (auto const & region : m_regions) @@ -100,6 +172,8 @@ void StreetsBuilder::AddStreetHighway(FeatureBuilder & fb) auto & street = InsertStreet(region.first, fb.GetName(), fb.GetMultilangName()); auto const osmId = pathSegments.size() == 1 ? fb.GetMostGenericOsmId() : NextOsmSurrogateId(); street.m_geometry.AddHighwayLine(osmId, std::move(segment.m_path)); + + m_streetFeatures2Streets.emplace(fb.GetMostGenericOsmId(), &street); } } @@ -114,6 +188,8 @@ void StreetsBuilder::AddStreetArea(FeatureBuilder & fb) auto & street = InsertStreet(region->first, fb.GetName(), fb.GetMultilangName()); auto osmId = fb.GetMostGenericOsmId(); street.m_geometry.AddHighwayArea(osmId, fb.GetOuterGeometry()); + + m_streetFeatures2Streets.emplace(osmId, &street); } void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb) @@ -127,6 +203,8 @@ void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb) auto osmId = fb.GetMostGenericOsmId(); auto & street = InsertStreet(region->first, fb.GetName(), fb.GetMultilangName()); street.m_geometry.SetPin({fb.GetKeyPoint(), osmId}); + + m_streetFeatures2Streets.emplace(osmId, &street); } void StreetsBuilder::AddStreetBinding(std::string && streetName, FeatureBuilder & fb, diff --git a/generator/streets/streets_builder.hpp b/generator/streets/streets_builder.hpp index 5279efc..7e7ae5e 100644 --- a/generator/streets/streets_builder.hpp +++ b/generator/streets/streets_builder.hpp @@ -1,6 +1,7 @@ #pragma once #include "generator/feature_builder.hpp" +#include "generator/feature_generator.hpp" #include "generator/key_value_storage.hpp" #include "generator/osm_element.hpp" #include "generator/regions/region_info_getter.hpp" @@ -12,10 +13,10 @@ #include "base/geo_object_id.hpp" -#include #include #include #include +#include #include #include @@ -32,6 +33,9 @@ public: void AssembleStreets(std::string const & pathInStreetsTmpMwm); void AssembleBindings(std::string const & pathInGeoObjectsTmpMwm); + + void RegenerateAggreatedStreetsFeatures(std::string const & pathStreetsTmpMwm); + // Save built streets in the jsonl format with the members: "properties", "bbox" (array: left // bottom longitude, left bottom latitude, right top longitude, right top latitude), "pin" (array: // longitude, latitude). @@ -48,6 +52,9 @@ private: }; using RegionStreets = std::unordered_map; + void WriteAsAggregatedStreet(feature::FeatureBuilder & fb, Street const & street, + feature::FeaturesCollector & collector) const; + void SaveRegionStreetsKv(std::ostream & streamStreetsKv, uint64_t regionId, RegionStreets const & streets); @@ -67,6 +74,7 @@ private: base::GeoObjectId NextOsmSurrogateId(); std::unordered_map m_regions; + std::unordered_multimap m_streetFeatures2Streets; regions::RegionInfoGetter const & m_regionInfoGetter; uint64_t m_osmSurrogateCounter{0}; size_t m_threadsCount; diff --git a/indexer/interval_index_builder.hpp b/indexer/interval_index_builder.hpp index 043306d..fdec757 100644 --- a/indexer/interval_index_builder.hpp +++ b/indexer/interval_index_builder.hpp @@ -240,6 +240,8 @@ public: { uint64_t const key = it->GetCell(); Value const value = it->GetValue(); + if (key == prevKey && value == prevValue) + continue; if (it != beg && (key >> skipBits) != (prevKey >> skipBits)) { sizes.push_back(writer.Pos() - prevPos); diff --git a/indexer/locality_object.cpp b/indexer/locality_object.cpp index a8263cc..67d784f 100644 --- a/indexer/locality_object.cpp +++ b/indexer/locality_object.cpp @@ -21,7 +21,17 @@ void LocalityObject::Deserialize(char const * data) return; } - ASSERT_EQUAL(type, feature::GeomType::Area, ("Only supported types are Point and Area.")); + if (type == feature::GeomType::Line) + { + uint32_t ptsCount; + ReadPrimitiveFromSource(src, ptsCount); + CHECK_GREATER(ptsCount, 1, ()); + char const * start = src.PtrC(); + src = ArrayByteSource(serial::LoadInnerPath(start, ptsCount, cp, m_points)); + return; + } + + ASSERT_EQUAL(type, feature::GeomType::Area, ()); uint32_t trgCount; ReadPrimitiveFromSource(src, trgCount); CHECK_GREATER(trgCount, 0, ()); diff --git a/indexer/locality_object.hpp b/indexer/locality_object.hpp index 27c50b5..e489722 100644 --- a/indexer/locality_object.hpp +++ b/indexer/locality_object.hpp @@ -64,7 +64,7 @@ public: buffer_vector strip; auto const index = FindSingleStrip( - m_points.size(), IsDiagonalVisibleFunctor::const_iterator>( + m_points.size(), IsDiagonalVisibleFunctor::const_iterator>( m_points.begin(), m_points.end())); MakeSingleStripFromIndex(index, m_points.size(), [&](size_t i) { strip.push_back(m_points[i]); }); @@ -73,7 +73,7 @@ public: private: uint64_t m_id = 0; - std::vector m_points; + buffer_vector m_points; // m_triangles[3 * i], m_triangles[3 * i + 1], m_triangles[3 * i + 2] form the i-th triangle. buffer_vector m_triangles; };