organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Add streets into geo_objects reverse index

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-10-08 11:57:16 +03:00
parent 4f5d44b8c4
commit e73e8f247f
13 changed files with 190 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
generator/feature_builder.cpp
View file

@ -649,7 +649,16 @@ void FeatureBuilder::SerializeLocalityObject(serial::GeometryCodingParams const
return;
}
CHECK_EQUAL(type, GeomType::Area, ("Supported types are Point and Area"));
if (type == GeomType::Line)
{
uint32_t const ptsCount = base::asserted_cast<uint32_t>(data.m_innerPts.size());
CHECK_GREATER(ptsCount, 1, ());
WriteToSink(sink, ptsCount);
serial::SaveInnerPath(data.m_innerPts, params, sink);
return;
}
CHECK_EQUAL(type, GeomType::Area, ());
uint32_t trgCount = base::asserted_cast<uint32_t>(data.m_innerTrg.size());
CHECK_GREATER(trgCount, 2, ());

1
generator/generator_tool/generator_tool.cpp
View file

@ -317,6 +317,7 @@ int GeneratorToolMain(int argc, char ** argv)
if (options.m_generate_geo_objects_index)
{
if (!feature::GenerateGeoObjectsData(options.m_geo_objects_features,
options.m_streets_features,
options.m_nodes_list_path, locDataFile))
{
LOG(LCRITICAL, ("Error generating geo objects data."));

3
generator/geo_objects/geo_objects.cpp
View file

@ -212,7 +212,8 @@ boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
{
auto const dataFile = GetPlatform().TmpPathForFile();
SCOPE_GUARD(removeDataFile, std::bind(Platform::RemoveFileIfExists, std::cref(dataFile)));
if (!GenerateGeoObjectsData(pathToGeoObjectsTmpMwm, "" /* nodesFile */, dataFile))
if (!GenerateGeoObjectsData(pathToGeoObjectsTmpMwm, "" /* streetFeaturesFile */,
"" /* nodesFile */, dataFile))
{
LOG(LCRITICAL, ("Error generating geo objects data."));
return {};

31
generator/locality_sorter.cpp
View file

@ -2,6 +2,7 @@
#include "generator/geo_objects/geo_objects_filter.hpp"
#include "generator/geometry_holder.hpp"
#include "generator/streets/streets_filter.hpp"
#include "generator/utils.hpp"
#include "indexer/data_header.hpp"
@ -113,6 +114,9 @@ public:
SimplifyPoints(distFn, scales::GetUpperScale(), holder.GetSourcePoints(), points);
if (fb.IsLine())
holder.AddPoints(points, 0);
// For areas we save outer geometry only.
if (fb.IsArea() && holder.NeedProcessTriangles())
{
@ -235,22 +239,39 @@ bool GenerateLocalityDataImpl(FeaturesCollector & collector,
namespace feature
{
bool GenerateGeoObjectsData(string const & featuresFile, string const & nodesFile,
string const & dataFile)
bool GenerateGeoObjectsData(string const & geoObjectsFeaturesFile,
string const & streetFeaturesFile,
string const & nodesFile, string const & dataFile)
{
auto featuresFile = geoObjectsFeaturesFile;
auto const geoObjectsAndStreetsFeaturesFile = GetPlatform().TmpPathForFile();
SCOPE_GUARD(geoObjectsAndStreetsFeaturesFileGuard,
std::bind(Platform::RemoveFileIfExists, geoObjectsAndStreetsFeaturesFile));
if (!streetFeaturesFile.empty())
{
auto features = std::ofstream{geoObjectsAndStreetsFeaturesFile, std::ios_base::binary};
for (auto const & file : {geoObjectsFeaturesFile , streetFeaturesFile})
{
auto fileStream = std::ifstream{file, std::ios_base::binary};
features << fileStream.rdbuf();
}
featuresFile = geoObjectsAndStreetsFeaturesFile;
}
set<uint64_t> nodeIds;
if (!ParseNodes(nodesFile, nodeIds))
return false;
auto const needSerialize = [&nodeIds](FeatureBuilder & fb) {
if (!fb.IsPoint() && !fb.IsArea())
return false;
using generator::geo_objects::GeoObjectsFilter;
using generator::streets::StreetsFilter;
if (GeoObjectsFilter::IsBuilding(fb) || GeoObjectsFilter::HasHouse(fb))
return true;
if (StreetsFilter::IsStreet(fb))
return true;
if (GeoObjectsFilter::IsPoi(fb))
return 0 != nodeIds.count(fb.GetMostGenericOsmId().GetEncodedId());

4
generator/locality_sorter.hpp
View file

@ -8,7 +8,9 @@ namespace feature
// @param featuresDir - path to folder with pregenerated features data;
// @param nodesFile - path to file with list of node ids we need to add to output;
// @param out - output file name;
bool GenerateGeoObjectsData(std::string const & featuresFile, std::string const & nodesFile,
bool GenerateGeoObjectsData(std::string const & geoObjectsFeaturesFile,
std::string const & streetFeaturesFile,
std::string const & nodesFile,
std::string const & out);
// Generates data for RegionsIndexBuilder from input feature-dat-files.

39
generator/streets/street_geometry.cpp
View file

@ -51,6 +51,16 @@ void StreetGeometry::SetPin(Pin && pin)
m_pin = std::move(pin);
}
boost::optional<Pin> const & StreetGeometry::GetPin() const
{
return m_pin;
}
HighwayGeometry const * StreetGeometry::GetHighwayGeometry() const
{
return m_highwayGeometry.get();
}
void StreetGeometry::AddHighwayLine(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & line)
{
if (!m_highwayGeometry)
@ -85,6 +95,21 @@ Pin HighwayGeometry::ChoosePin() const
return ChooseMultilinePin();
}
m2::RectD const & HighwayGeometry::GetBbox() const
{
return m_limitRect;
}
std::vector<HighwayGeometry::AreaPart> const & HighwayGeometry::GetAreaParts() const
{
return m_areaParts;
}
HighwayGeometry::MultiLine const & HighwayGeometry::GetMultiLine() const
{
return m_multiLine;
}
Pin HighwayGeometry::ChooseMultilinePin() const
{
auto const & lines = m_multiLine.m_lines;
@ -145,11 +170,6 @@ void HighwayGeometry::AddArea(base::GeoObjectId const & osmId, std::vector<m2::P
ExtendLimitRect(border);
}
m2::RectD const & HighwayGeometry::GetBbox() const
{
return m_limitRect;
}
void HighwayGeometry::ExtendLimitRect(std::vector<m2::PointD> const & points)
{
feature::CalcRect(points, m_limitRect);
@ -315,13 +335,14 @@ double HighwayGeometry::LineSegment::CalculateLength() const noexcept
// HighwayGeometry::AreaPart ---------------------------------------------------------------------------------
HighwayGeometry::AreaPart::AreaPart(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & polygon)
: m_osmId{osmId}
HighwayGeometry::AreaPart::AreaPart(base::GeoObjectId const & osmId,
std::vector<m2::PointD> const & border)
: m_osmId{osmId}, m_border{border}
{
CHECK_GREATER_OR_EQUAL(polygon.size(), 3, ());
CHECK_GREATER_OR_EQUAL(border.size(), 3, ());
auto boostPolygon = boost_helpers::BoostPolygon{};
for (auto const & p : polygon)
for (auto const & p : border)
boost::geometry::append(boostPolygon, boost_helpers::BoostPoint{p.x, p.y});
boost::geometry::correct(boostPolygon);

21
generator/streets/street_geometry.hpp
View file

@ -32,6 +32,8 @@ public:
Pin GetOrChoosePin() const;
// Bbox may be the minimum bounding box with feature type margin.
m2::RectD GetBbox() const;
boost::optional<Pin> const & GetPin() const;
HighwayGeometry const * GetHighwayGeometry() const;
void SetPin(Pin && pin);
void AddHighwayLine(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & line);
@ -49,13 +51,6 @@ private:
class HighwayGeometry
{
public:
Pin ChoosePin() const;
m2::RectD const & GetBbox() const;
void AddLine(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & line);
void AddArea(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & border);
private:
struct LineSegment
{
LineSegment(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & points);
@ -93,13 +88,23 @@ private:
struct AreaPart
{
AreaPart(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & polygon);
AreaPart(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & border);
base::GeoObjectId m_osmId;
std::vector<m2::PointD> m_border;
m2::PointD m_center;
double m_area;
};
Pin ChoosePin() const;
m2::RectD const & GetBbox() const;
std::vector<AreaPart> const & GetAreaParts() const;
MultiLine const & GetMultiLine() const;
void AddLine(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & line);
void AddArea(base::GeoObjectId const & osmId, std::vector<m2::PointD> const & border);
private:
Pin ChooseMultilinePin() const;
Pin ChooseLinePin(Line const & line, double disposeDistance) const;
Pin ChooseAreaPin() const;

3
generator/streets/streets.cpp
View file

@ -36,6 +36,9 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const &
streetsBuilder.AssembleBindings(pathInGeoObjectsTmpMwm);
LOG(LINFO, ("Binding's streets were built."));
streetsBuilder.RegenerateAggreatedStreetsFeatures(pathInStreetsTmpMwm);
LOG(LINFO, ("Streets features are aggreated into", pathInStreetsTmpMwm));
std::ofstream streamStreetsKv(pathOutStreetsKv);
streetsBuilder.SaveStreetsKv(streamStreetsKv);
LOG(LINFO, ("Streets key-value storage saved to", pathOutStreetsKv));

78
generator/streets/streets_builder.cpp
View file

@ -1,14 +1,20 @@
#include "generator/streets/streets_builder.hpp"
#include "generator/key_value_storage.hpp"
#include "generator/streets/street_regions_tracing.hpp"
#include "generator/translation.hpp"
#include "coding/internal/file_data.hpp"
#include "indexer/classificator.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "geometry/mercator.hpp"
#include "platform/platform.hpp"
#include "base/logging.hpp"
#include "base/scope_guard.hpp"
#include <utility>
@ -47,6 +53,72 @@ void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm
ForEachParallelFromDatRawFormat(m_threadsCount, pathInGeoObjectsTmpMwm, transform);
}
void StreetsBuilder::RegenerateAggreatedStreetsFeatures(
std::string const & pathStreetsTmpMwm)
{
auto const aggregatedStreetsTmpFile = GetPlatform().TmpPathForFile();
SCOPE_GUARD(aggregatedStreetsTmpFileGuard,
std::bind(Platform::RemoveFileIfExists, aggregatedStreetsTmpFile));
FeaturesCollector collector(aggregatedStreetsTmpFile);
std::set<Street const *> processedStreets;
auto const transform = [&](FeatureBuilder & fb, uint64_t /* currPos */) {
auto street = m_streetFeatures2Streets.find(fb.GetMostGenericOsmId());
if (street == m_streetFeatures2Streets.end())
return;
if (!processedStreets.insert(street->second).second)
return;
WriteAsAggregatedStreet(fb, *street->second, collector);
};
ForEachFromDatRawFormat(pathStreetsTmpMwm, transform);
CHECK(base::RenameFileX(aggregatedStreetsTmpFile, pathStreetsTmpMwm), ());
}
void StreetsBuilder::WriteAsAggregatedStreet(FeatureBuilder & fb, Street const & street,
FeaturesCollector & collector) const
{
fb.GetParams().name = street.m_name;
auto const & geometry = street.m_geometry;
auto const & pin = geometry.GetOrChoosePin();
fb.SetOsmId(pin.m_osmId);
if (auto const & pin = geometry.GetPin())
{
fb.ResetGeometry();
fb.SetCenter(pin->m_position);
collector.Collect(fb);
}
auto const * highwayGeometry = geometry.GetHighwayGeometry();
if (!highwayGeometry)
return;
for (auto const & area : highwayGeometry->GetAreaParts())
{
fb.ResetGeometry();
fb.GetParams().SetGeomType(feature::GeomType::Area);
auto polygon = area.m_border;
fb.AddPolygon(polygon);
collector.Collect(fb);
}
for (auto const & line : highwayGeometry->GetMultiLine().m_lines)
{
for (auto const & segment : line.m_segments)
{
fb.ResetGeometry();
fb.SetLinear();
for (auto const & point : segment.m_points)
fb.AddPoint(point);
collector.Collect(fb);
}
}
}
void StreetsBuilder::SaveStreetsKv(std::ostream & streamStreetsKv)
{
for (auto const & region : m_regions)
@ -100,6 +172,8 @@ void StreetsBuilder::AddStreetHighway(FeatureBuilder & fb)
auto & street = InsertStreet(region.first, fb.GetName(), fb.GetMultilangName());
auto const osmId = pathSegments.size() == 1 ? fb.GetMostGenericOsmId() : NextOsmSurrogateId();
street.m_geometry.AddHighwayLine(osmId, std::move(segment.m_path));
m_streetFeatures2Streets.emplace(fb.GetMostGenericOsmId(), &street);
}
}
@ -114,6 +188,8 @@ void StreetsBuilder::AddStreetArea(FeatureBuilder & fb)
auto & street = InsertStreet(region->first, fb.GetName(), fb.GetMultilangName());
auto osmId = fb.GetMostGenericOsmId();
street.m_geometry.AddHighwayArea(osmId, fb.GetOuterGeometry());
m_streetFeatures2Streets.emplace(osmId, &street);
}
void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb)
@ -127,6 +203,8 @@ void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb)
auto osmId = fb.GetMostGenericOsmId();
auto & street = InsertStreet(region->first, fb.GetName(), fb.GetMultilangName());
street.m_geometry.SetPin({fb.GetKeyPoint(), osmId});
m_streetFeatures2Streets.emplace(osmId, &street);
}
void StreetsBuilder::AddStreetBinding(std::string && streetName, FeatureBuilder & fb,

10
generator/streets/streets_builder.hpp
View file

@ -1,6 +1,7 @@
#pragma once
#include "generator/feature_builder.hpp"
#include "generator/feature_generator.hpp"
#include "generator/key_value_storage.hpp"
#include "generator/osm_element.hpp"
#include "generator/regions/region_info_getter.hpp"
@ -12,10 +13,10 @@
#include "base/geo_object_id.hpp"
#include <stdint.h>
#include <memory>
#include <mutex>
#include <ostream>
#include <stdint.h>
#include <string>
#include <unordered_map>
@ -32,6 +33,9 @@ public:
void AssembleStreets(std::string const & pathInStreetsTmpMwm);
void AssembleBindings(std::string const & pathInGeoObjectsTmpMwm);
void RegenerateAggreatedStreetsFeatures(std::string const & pathStreetsTmpMwm);
// Save built streets in the jsonl format with the members: "properties", "bbox" (array: left
// bottom longitude, left bottom latitude, right top longitude, right top latitude), "pin" (array:
// longitude, latitude).
@ -48,6 +52,9 @@ private:
};
using RegionStreets = std::unordered_map<std::string, Street>;
void WriteAsAggregatedStreet(feature::FeatureBuilder & fb, Street const & street,
feature::FeaturesCollector & collector) const;
void SaveRegionStreetsKv(std::ostream & streamStreetsKv, uint64_t regionId,
RegionStreets const & streets);
@ -67,6 +74,7 @@ private:
base::GeoObjectId NextOsmSurrogateId();
std::unordered_map<uint64_t, RegionStreets> m_regions;
std::unordered_multimap<base::GeoObjectId, Street const *> m_streetFeatures2Streets;
regions::RegionInfoGetter const & m_regionInfoGetter;
uint64_t m_osmSurrogateCounter{0};
size_t m_threadsCount;

2
indexer/interval_index_builder.hpp
View file

@ -240,6 +240,8 @@ public:
{
uint64_t const key = it->GetCell();
Value const value = it->GetValue();
if (key == prevKey && value == prevValue)
continue;
if (it != beg && (key >> skipBits) != (prevKey >> skipBits))
{
sizes.push_back(writer.Pos() - prevPos);

12
indexer/locality_object.cpp
View file

@ -21,7 +21,17 @@ void LocalityObject::Deserialize(char const * data)
return;
}
ASSERT_EQUAL(type, feature::GeomType::Area, ("Only supported types are Point and Area."));
if (type == feature::GeomType::Line)
{
uint32_t ptsCount;
ReadPrimitiveFromSource(src, ptsCount);
CHECK_GREATER(ptsCount, 1, ());
char const * start = src.PtrC();
src = ArrayByteSource(serial::LoadInnerPath(start, ptsCount, cp, m_points));
return;
}
ASSERT_EQUAL(type, feature::GeomType::Area, ());
uint32_t trgCount;
ReadPrimitiveFromSource(src, trgCount);
CHECK_GREATER(trgCount, 0, ());

4
indexer/locality_object.hpp
View file

@ -64,7 +64,7 @@ public:
buffer_vector<m2::PointD, 32> strip;
auto const index = FindSingleStrip(
m_points.size(), IsDiagonalVisibleFunctor<std::vector<m2::PointD>::const_iterator>(
m_points.size(), IsDiagonalVisibleFunctor<buffer_vector<m2::PointD, 32>::const_iterator>(
m_points.begin(), m_points.end()));
MakeSingleStripFromIndex(index, m_points.size(),
[&](size_t i) { strip.push_back(m_points[i]); });
@ -73,7 +73,7 @@ public:
private:
uint64_t m_id = 0;
std::vector<m2::PointD> m_points;
buffer_vector<m2::PointD, 32> m_points;
// m_triangles[3 * i], m_triangles[3 * i + 1], m_triangles[3 * i + 2] form the i-th triangle.
buffer_vector<m2::PointD, 32> m_triangles;
};