From 61fd4acdb53e63d715cd641b08ffa67fe5e249ed Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Fri, 20 Jul 2018 21:11:10 +0300 Subject: [PATCH] [generator] Changed logic for building mwm for regions --- base/CMakeLists.txt | 1 + base/utils.hpp | 9 + generator/CMakeLists.txt | 2 + generator/intermediate_data.cpp | 98 +++++- generator/intermediate_data.hpp | 135 ++++---- generator/osm_source.cpp | 316 +++++++----------- generator/osm_source.hpp | 4 +- generator/osm_translator.cpp | 552 ++++++++++++++++++++++++++++++++ generator/osm_translator.hpp | 539 +++++-------------------------- generator/ways_merger.cpp | 20 ++ generator/ways_merger.hpp | 28 +- 11 files changed, 957 insertions(+), 747 deletions(-) create mode 100644 base/utils.hpp create mode 100644 generator/osm_translator.cpp create mode 100644 generator/ways_merger.cpp diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 00fec2ab05..232c400cd6 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -93,6 +93,7 @@ set( uni_string_dfa.hpp url_helpers.cpp url_helpers.hpp + utils.hpp visitor.hpp worker_thread.cpp worker_thread.hpp diff --git a/base/utils.hpp b/base/utils.hpp new file mode 100644 index 0000000000..d1e127b8e9 --- /dev/null +++ b/base/utils.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include + +template +inline bool TestOverflow(V value) +{ + return value >= std::numeric_limits::min() && value <= std::numeric_limits::max(); +} diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index f3cbd46138..39707b3d79 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -60,6 +60,7 @@ set(SRC osm_o5m_source.hpp osm_source.cpp osm_translator.hpp + osm_translator.cpp osm_xml_source.hpp polygonizer.hpp popular_places_section_builder.cpp @@ -111,6 +112,7 @@ set(SRC viator_dataset.cpp viator_dataset.hpp ways_merger.hpp + ways_merger.cpp world_map_generator.hpp ) diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index 358f1b4aa2..f4ccf4b612 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -5,6 +5,7 @@ #include "base/assert.hpp" #include "base/logging.hpp" +#include "base/utils.hpp" #include "defines.hpp" @@ -26,12 +27,8 @@ void ToLatLon(double lat, double lon, generator::cache::LatLon & ll) int64_t const lat64 = lat * kValueOrder; int64_t const lon64 = lon * kValueOrder; - CHECK( - lat64 >= std::numeric_limits::min() && lat64 <= std::numeric_limits::max(), - ("Latitude is out of 32bit boundary:", lat64)); - CHECK( - lon64 >= std::numeric_limits::min() && lon64 <= std::numeric_limits::max(), - ("Longitude is out of 32bit boundary:", lon64)); + CHECK(TestOverflow(lat64), ("Latitude is out of 32bit boundary:", lat64)); + CHECK(TestOverflow(lon64), ("Longitude is out of 32bit boundary:", lon64)); ll.m_lat = static_cast(lat64); ll.m_lon = static_cast(lon64); } @@ -266,5 +263,94 @@ void MapFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon) ++m_numProcessedPoints; } + +// IntermediateDataReader +IntermediateDataReader::IntermediateDataReader(shared_ptr nodes, + feature::GenerateInfo & info) : + m_nodes(nodes), + m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache), + m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache), + m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)), + m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) +{ +} + +void IntermediateDataReader::LoadIndex() +{ + m_ways.LoadOffsets(); + m_relations.LoadOffsets(); + + m_nodeToRelations.ReadAll(); + m_wayToRelations.ReadAll(); +} + + +// IntermediateDataWriter +IntermediateDataWriter::IntermediateDataWriter(std::shared_ptr nodes, + feature::GenerateInfo & info): + m_nodes(nodes), + m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache), + m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache), + m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)), + m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) +{ +} + +void IntermediateDataWriter::AddRelation(Key id, RelationElement const & e) +{ + string const & relationType = e.GetType(); + if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" || + relationType == "associatedStreet" || relationType == "building" || + relationType == "restriction")) + { + return; + } + + m_relations.Write(id, e); + AddToIndex(m_nodeToRelations, id, e.nodes); + AddToIndex(m_wayToRelations, id, e.ways); +} + +void IntermediateDataWriter::SaveIndex() +{ + m_ways.SaveOffsets(); + m_relations.SaveOffsets(); + + m_nodeToRelations.WriteAll(); + m_wayToRelations.WriteAll(); +} + + +// Functions +std::shared_ptr +CreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, string const & name) +{ + switch (type) + { + case feature::GenerateInfo::NodeStorageType::File: + return std::make_shared(name); + case feature::GenerateInfo::NodeStorageType::Index: + return std::make_shared(name); + case feature::GenerateInfo::NodeStorageType::Memory: + return std::make_shared(name); + } + CHECK_SWITCH(); +} + +std::shared_ptr +CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::string const & name) +{ + switch (type) + { + case feature::GenerateInfo::NodeStorageType::File: + return std::make_shared(name); + case feature::GenerateInfo::NodeStorageType::Index: + return std::make_shared(name); + case feature::GenerateInfo::NodeStorageType::Memory: + return std::make_shared(name); + } + CHECK_SWITCH(); +} + } // namespace cache } // namespace generator diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index ddea7f2da8..dfd0b048ef 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -181,51 +181,64 @@ protected: bool m_preload = false; }; -class RawFilePointStorageMmapReader +class IPointStorageWriter +{ +public: + virtual void AddPoint(uint64_t id, double lat, double lon) = 0; + virtual uint64_t GetNumProcessedPoints() const = 0; +}; + +class IPointStorageReader +{ +public: + virtual bool GetPoint(uint64_t id, double & lat, double & lon) const = 0; +}; + +class RawFilePointStorageMmapReader : public IPointStorageReader { public: explicit RawFilePointStorageMmapReader(std::string const & name); - bool GetPoint(uint64_t id, double & lat, double & lon) const; + bool GetPoint(uint64_t id, double & lat, double & lon) const override; private: MmapReader m_mmapReader; }; -class RawFilePointStorageWriter +class RawFilePointStorageWriter : public IPointStorageWriter { public: explicit RawFilePointStorageWriter(std::string const & name); - void AddPoint(uint64_t id, double lat, double lon); - uint64_t GetNumProcessedPoints() const { return m_numProcessedPoints; } + void AddPoint(uint64_t id, double lat, double lon) override; + uint64_t GetNumProcessedPoints() const override { return m_numProcessedPoints; } private: FileWriter m_fileWriter; uint64_t m_numProcessedPoints = 0; }; -class RawMemPointStorageReader +class RawMemPointStorageReader : public IPointStorageReader { public: explicit RawMemPointStorageReader(std::string const & name); - bool GetPoint(uint64_t id, double & lat, double & lon) const; + bool GetPoint(uint64_t id, double & lat, double & lon) const override; private: FileReader m_fileReader; std::vector m_data; }; -class RawMemPointStorageWriter +class RawMemPointStorageWriter : public IPointStorageWriter { public: explicit RawMemPointStorageWriter(std::string const & name); ~RawMemPointStorageWriter(); - void AddPoint(uint64_t id, double lat, double lon); - uint64_t GetNumProcessedPoints() const { return m_numProcessedPoints; } + void AddPoint(uint64_t id, double lat, double lon) override; + uint64_t GetNumProcessedPoints() const override { return m_numProcessedPoints; } private: FileWriter m_fileWriter; @@ -233,46 +246,39 @@ private: uint64_t m_numProcessedPoints = 0; }; -class MapFilePointStorageReader +class MapFilePointStorageReader : public IPointStorageReader { public: explicit MapFilePointStorageReader(std::string const & name); - bool GetPoint(uint64_t id, double & lat, double & lon) const; + bool GetPoint(uint64_t id, double & lat, double & lon) const override; private: FileReader m_fileReader; std::unordered_map m_map; }; -class MapFilePointStorageWriter +class MapFilePointStorageWriter : public IPointStorageWriter { public: explicit MapFilePointStorageWriter(std::string const & name); - void AddPoint(uint64_t id, double lat, double lon); - uint64_t GetNumProcessedPoints() const { return m_numProcessedPoints; } + void AddPoint(uint64_t id, double lat, double lon) override; + uint64_t GetNumProcessedPoints() const override { return m_numProcessedPoints; } private: FileWriter m_fileWriter; uint64_t m_numProcessedPoints = 0; }; -template class IntermediateDataReader { public: - IntermediateDataReader(NodesHolder & nodes, feature::GenerateInfo & info) - : m_nodes(nodes) - , m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache) - , m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache) - , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) - , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) - { - } - - bool GetNode(Key id, double & lat, double & lon) { return m_nodes.GetPoint(id, lat, lon); } + explicit IntermediateDataReader(shared_ptr nodes, + feature::GenerateInfo & info); + bool GetNode(Key id, double & lat, double & lon) { return m_nodes->GetPoint(id, lat, lon); } bool GetWay(Key id, WayElement & e) { return m_ways.Read(id, e); } + void LoadIndex(); template void ForEachRelationByWay(Key id, ToDo && toDo) @@ -295,15 +301,6 @@ public: m_nodeToRelations.ForEachByKey(id, processor); } - void LoadIndex() - { - m_ways.LoadOffsets(); - m_relations.LoadOffsets(); - - m_nodeToRelations.ReadAll(); - m_wayToRelations.ReadAll(); - } - private: using CacheReader = cache::OSMElementCacheReader; @@ -341,7 +338,8 @@ private: bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } }; - NodesHolder & m_nodes; +private: + shared_ptr m_nodes; cache::OSMElementCacheReader m_ways; cache::OSMElementCacheReader m_relations; @@ -350,62 +348,39 @@ private: cache::IndexFileReader m_wayToRelations; }; -template class IntermediateDataWriter { public: - IntermediateDataWriter(NodesHolder & nodes, feature::GenerateInfo & info) - : m_nodes(nodes) - , m_ways(info.GetIntermediateFileName(WAYS_FILE, ""), info.m_preloadCache) - , m_relations(info.GetIntermediateFileName(RELATIONS_FILE, ""), info.m_preloadCache) - , m_nodeToRelations(info.GetIntermediateFileName(NODES_FILE, ID2REL_EXT)) - , m_wayToRelations(info.GetIntermediateFileName(WAYS_FILE, ID2REL_EXT)) - { - } - - void AddNode(Key id, double lat, double lon) { m_nodes.AddPoint(id, lat, lon); } - + explicit IntermediateDataWriter(std::shared_ptr nodes, + feature::GenerateInfo & info); + void AddNode(Key id, double lat, double lon) { m_nodes->AddPoint(id, lat, lon); } void AddWay(Key id, WayElement const & e) { m_ways.Write(id, e); } - - void AddRelation(Key id, RelationElement const & e) - { - string const & relationType = e.GetType(); - if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary" || - relationType == "associatedStreet" || relationType == "building" || - relationType == "restriction")) - { - return; - } - - m_relations.Write(id, e); - AddToIndex(m_nodeToRelations, id, e.nodes); - AddToIndex(m_wayToRelations, id, e.ways); - } - - void SaveIndex() - { - m_ways.SaveOffsets(); - m_relations.SaveOffsets(); - - m_nodeToRelations.WriteAll(); - m_wayToRelations.WriteAll(); - } + void AddRelation(Key id, RelationElement const & e); + void SaveIndex(); private: - NodesHolder & m_nodes; - - cache::OSMElementCacheWriter m_ways; - cache::OSMElementCacheWriter m_relations; - - cache::IndexFileWriter m_nodeToRelations; - cache::IndexFileWriter m_wayToRelations; - template static void AddToIndex(Index & index, Key relationId, Container const & values) { for (auto const & v : values) index.Add(v.first, relationId); } + +private: + std::shared_ptr m_nodes; + + cache::OSMElementCacheWriter m_ways; + cache::OSMElementCacheWriter m_relations; + + cache::IndexFileWriter m_nodeToRelations; + cache::IndexFileWriter m_wayToRelations; }; + +std::shared_ptr +CreatePointStorageReader(feature::GenerateInfo::NodeStorageType type, std::string const & name); + +std::shared_ptr +CreatePointStorageWriter(feature::GenerateInfo::NodeStorageType type, std::string const & name); + } // namespace cache } // namespace generator diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 0604e68c8b..c3d5c7f177 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -184,7 +184,7 @@ public: Classificator const & c = classif(); char const * arr[][2] = { - {"natural", "coastline"}, {"natural", "land"}, {"place", "island"}, {"place", "islet"}}; + {"natural", "coastline"}, {"natural", "land"}, {"place", "island"}, {"place", "islet"}}; static_assert(ARRAY_SIZE(arr) == TYPES_COUNT, ""); for (size_t i = 0; i < ARRAY_SIZE(arr); ++i) @@ -200,14 +200,14 @@ public: m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE))); m_coastsHolder.reset(new feature::FeaturesAndRawGeometryCollector( - m_srcCoastsFile, - info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION))); + m_srcCoastsFile, + info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION))); return; } if (info.m_emitCoasts) m_coastsHolder.reset( - new feature::FeaturesCollector(info.GetTmpFileName(WORLD_COASTS_FILE_NAME))); + new feature::FeaturesCollector(info.GetTmpFileName(WORLD_COASTS_FILE_NAME))); if (info.m_splitByPolygons || !info.m_fileName.empty()) m_countries = my::make_unique(info); @@ -238,8 +238,8 @@ public: Place const place(fb, type); UnionEqualPlacesIds(place); m_places.ReplaceEqualInRect( - place, [](Place const & p1, Place const & p2) { return p1.IsEqual(p2); }, - [](Place const & p1, Place const & p2) { return p1.IsBetterThan(p2); }); + place, [](Place const & p1, Place const & p2) { return p1.IsEqual(p2); }, + [](Place const & p1, Place const & p2) { return p1.IsBetterThan(p2); }); return; } @@ -452,10 +452,10 @@ uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize) } // Functions --------------------------------------------------------------------------------------- -unique_ptr MakeMainFeatureEmitter(feature::GenerateInfo const & info) +shared_ptr MakeMainFeatureEmitter(feature::GenerateInfo const & info) { LOG(LINFO, ("Processing booking data from", info.m_bookingDatafileName, "done.")); - return my::make_unique(info); + return make_shared(info); } template @@ -463,46 +463,46 @@ void AddElementToCache(TCache & cache, TElement const & em) { switch (em.type) { - case TElement::EntityType::Node: + case TElement::EntityType::Node: + { + auto const pt = MercatorBounds::FromLatLon(em.lat, em.lon); + cache.AddNode(em.id, pt.y, pt.x); + break; + } + case TElement::EntityType::Way: + { + // store way + WayElement way(em.id); + for (uint64_t nd : em.Nodes()) + way.nodes.push_back(nd); + + if (way.IsValid()) + cache.AddWay(em.id, way); + break; + } + case TElement::EntityType::Relation: + { + // store relation + RelationElement relation; + for (auto const & member : em.Members()) { - auto const pt = MercatorBounds::FromLatLon(em.lat, em.lon); - cache.AddNode(em.id, pt.y, pt.x); - break; + if (member.type == TElement::EntityType::Node) + relation.nodes.emplace_back(make_pair(member.ref, string(member.role))); + else if (member.type == TElement::EntityType::Way) + relation.ways.emplace_back(make_pair(member.ref, string(member.role))); + // we just ignore type == "relation" } - case TElement::EntityType::Way: - { - // store way - WayElement way(em.id); - for (uint64_t nd : em.Nodes()) - way.nodes.push_back(nd); - if (way.IsValid()) - cache.AddWay(em.id, way); - break; - } - case TElement::EntityType::Relation: - { - // store relation - RelationElement relation; - for (auto const & member : em.Members()) - { - if (member.type == TElement::EntityType::Node) - relation.nodes.emplace_back(make_pair(member.ref, string(member.role))); - else if (member.type == TElement::EntityType::Way) - relation.ways.emplace_back(make_pair(member.ref, string(member.role))); - // we just ignore type == "relation" - } + for (auto const & tag : em.Tags()) + relation.tags.emplace(make_pair(string(tag.key), string(tag.value))); - for (auto const & tag : em.Tags()) - relation.tags.emplace(make_pair(string(tag.key), string(tag.value))); + if (relation.IsValid()) + cache.AddRelation(em.id, relation); - if (relation.IsValid()) - cache.AddRelation(em.id, relation); - - break; - } - default: - break; + break; + } + default: + break; } } @@ -510,10 +510,10 @@ template void BuildIntermediateDataFromXML(SourceReader & stream, TCache & cache, TownsDumper & towns) { XMLSource parser([&](OsmElement * e) - { - towns.CheckElement(*e); - AddElementToCache(cache, *e); - }); + { + towns.CheckElement(*e); + AddElementToCache(cache, *e); + }); ParseXMLSequence(stream, parser); } @@ -564,28 +564,28 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function; -template -bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter, - PreEmit const & preEmit) +static bool GenerateRaw(feature::GenerateInfo & info, std::shared_ptr emitter, + PreEmit const & preEmit, std::unique_ptr parser) { try { - NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - - using TDataCache = cache::IntermediateDataReader; - TDataCache cache(nodes, info); - cache.LoadIndex(); - - // TODO(mgsergio): Get rid of EmitterBase template parameter. - OsmToFeatureTranslator parser( - emitter, cache, info.m_makeCoasts ? classif().GetCoastType() : 0, - info.GetAddressesFileName(), info.GetIntermediateFileName(RESTRICTIONS_FILENAME, ""), - info.GetIntermediateFileName(ROAD_ACCESS_FILENAME, ""), - info.GetIntermediateFileName(METALINES_FILENAME, "")); - auto const fn = [&](OsmElement * e) { if (preEmit(e)) - parser.EmitElement(e); + parser->EmitElement(e); }; SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); switch (info.m_osmFileType) { - case feature::GenerateInfo::OsmSourceType::XML: - ProcessOsmElementsFromXML(reader, fn); - break; - case feature::GenerateInfo::OsmSourceType::O5M: - ProcessOsmElementsFromO5M(reader, fn); - break; + case feature::GenerateInfo::OsmSourceType::XML: + ProcessOsmElementsFromXML(reader, fn); + break; + case feature::GenerateInfo::OsmSourceType::O5M: + ProcessOsmElementsFromO5M(reader, fn); + break; } LOG(LINFO, ("Processing", info.m_osmFileName, "done.")); @@ -641,10 +627,10 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter, generator::MixFakeNodes(GetPlatform().ResourcesDir() + MIXED_NODES_FILE, fn); // Stop if coasts are not merged and FLAG_fail_on_coasts is set - if (!emitter.Finish()) + if (!emitter->Finish()) return false; - emitter.GetNames(info.m_bucketNames); + emitter->GetNames(info.m_bucketNames); } catch (Reader::Exception const & ex) { @@ -654,52 +640,13 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter, return true; } -template -bool GenerateIntermediateDataImpl(feature::GenerateInfo & info) +static cache::IntermediateDataReader LoadCache(feature::GenerateInfo & info) { - try - { - NodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - cache::IntermediateDataWriter cache(nodes, info); - TownsDumper towns; - - SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); - - LOG(LINFO, ("Data source:", info.m_osmFileName)); - - switch (info.m_osmFileType) - { - case feature::GenerateInfo::OsmSourceType::XML: - BuildIntermediateDataFromXML(reader, cache, towns); - break; - case feature::GenerateInfo::OsmSourceType::O5M: - BuildIntermediateDataFromO5M(reader, cache, towns); - break; - } - - cache.SaveIndex(); - towns.Dump(info.GetIntermediateFileName(TOWNS_FILE, "")); - LOG(LINFO, ("Added points count =", nodes.GetNumProcessedPoints())); - } - catch (Writer::Exception const & e) - { - LOG(LCRITICAL, ("Error with file:", e.what())); - } - return true; -} - -bool GenerateRaw(feature::GenerateInfo & info, std::unique_ptr emitter, PreEmit const & preEmit) -{ - switch (info.m_nodeStorageType) - { - case feature::GenerateInfo::NodeStorageType::File: - return GenerateFeaturesImpl(info, *emitter, preEmit); - case feature::GenerateInfo::NodeStorageType::Index: - return GenerateFeaturesImpl(info, *emitter, preEmit); - case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateFeaturesImpl(info, *emitter, preEmit); - } - return false; + auto nodes = cache::CreatePointStorageReader(info.m_nodeStorageType, + info.GetIntermediateFileName(NODES_FILE, "")); + cache::IntermediateDataReader cache(nodes, info); + cache.LoadIndex(); + return cache; } bool GenerateFeatures(feature::GenerateInfo & info, EmitterFactory factory) @@ -717,68 +664,53 @@ bool GenerateFeatures(feature::GenerateInfo & info, EmitterFactory factory) return true; }; - return GenerateRaw(info, factory(info), preEmit); + auto cache = LoadCache(info); + auto emitter = factory(info); + auto parser = std::make_unique(emitter, cache, info); + return GenerateRaw(info, emitter, preEmit, std::move(parser)); } bool GenerateRegionFeatures(feature::GenerateInfo & info, EmitterFactory factory) { - set const adminLevels = {"2", "4", "5", "6"}; - set const places = {"city", "town", "village", "hamlet", "suburb"}; - - auto isRegion = [&adminLevels, &places](OsmElement const & e) { - // We do not make any assumptions about shape of places without explicit border for now. - if (e.type != OsmElement::EntityType::Way && e.type != OsmElement::EntityType::Relation) - return false; - - bool haveBoundary = false; - bool haveAdminLevel = false; - for (auto const & t : e.Tags()) - { - if (t.key == "boundary" && t.value == "administrative") - haveBoundary = true; - - if (t.key == "admin_level" && adminLevels.find(t.value) != adminLevels.end()) - haveAdminLevel = true; - - if (haveBoundary && haveAdminLevel) - return true; - - if (t.key == "place" && places.find(t.value) != places.end()) - return true; - } - - return false; - }; - - auto preEmit = [&isRegion](OsmElement * e) { - if (isRegion(*e)) - { - // Emit feature with original geometry and visible "natural = land" tag. - // Now emitter does not have a single place of decision which elements to emit and which to - // ignore. So the only way to make it emit element is to construct "good" element. - // This code should be removed in case of emitter refactoring. - e->m_tags = {}; - e->AddTag("natural", "land"); - e->AddTag("type", "multipolygon"); - return true; - } - return false; - }; - - return GenerateRaw(info, factory(info), preEmit); + auto preEmit = [](OsmElement * e) { return true; }; + auto cache = LoadCache(info); + auto emitter = factory(info); + auto parser = std::make_unique(emitter, cache); + return GenerateRaw(info, emitter, preEmit, std::move(parser)); } bool GenerateIntermediateData(feature::GenerateInfo & info) { - switch (info.m_nodeStorageType) + try { - case feature::GenerateInfo::NodeStorageType::File: - return GenerateIntermediateDataImpl(info); - case feature::GenerateInfo::NodeStorageType::Index: - return GenerateIntermediateDataImpl(info); - case feature::GenerateInfo::NodeStorageType::Memory: - return GenerateIntermediateDataImpl(info); + auto nodes = cache::CreatePointStorageWriter(info.m_nodeStorageType, + info.GetIntermediateFileName(NODES_FILE, "")); + cache::IntermediateDataWriter cache(nodes, info); + TownsDumper towns; + + SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); + + LOG(LINFO, ("Data source:", info.m_osmFileName)); + + switch (info.m_osmFileType) + { + case feature::GenerateInfo::OsmSourceType::XML: + BuildIntermediateDataFromXML(reader, cache, towns); + break; + case feature::GenerateInfo::OsmSourceType::O5M: + BuildIntermediateDataFromO5M(reader, cache, towns); + break; + } + + cache.SaveIndex(); + towns.Dump(info.GetIntermediateFileName(TOWNS_FILE, "")); + LOG(LINFO, ("Added points count =", nodes->GetNumProcessedPoints())); } - return false; + catch (Writer::Exception const & e) + { + LOG(LCRITICAL, ("Error with file:", e.what())); + } + return true; } + } // namespace generator diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp index eb820cb1fd..3011f1ffa6 100644 --- a/generator/osm_source.hpp +++ b/generator/osm_source.hpp @@ -57,9 +57,9 @@ public: virtual void GetNames(std::vector & names) const = 0; }; -std::unique_ptr MakeMainFeatureEmitter(feature::GenerateInfo const & info); +std::shared_ptr MakeMainFeatureEmitter(feature::GenerateInfo const & info); -using EmitterFactory = std::function(feature::GenerateInfo const &)>; +using EmitterFactory = std::function(feature::GenerateInfo const &)>; bool GenerateFeatures(feature::GenerateInfo & info, EmitterFactory factory = MakeMainFeatureEmitter); diff --git a/generator/osm_translator.cpp b/generator/osm_translator.cpp new file mode 100644 index 0000000000..12d87ff10b --- /dev/null +++ b/generator/osm_translator.cpp @@ -0,0 +1,552 @@ +#include "generator/osm_translator.hpp" + +namespace generator { + +// RelationTagsBase +RelationTagsBase::RelationTagsBase(routing::TagsProcessor & tagsProcessor) + : m_routingTagsProcessor(tagsProcessor), m_cache(14) +{ +} + +void RelationTagsBase::Reset(uint64_t fID, OsmElement * p) +{ + m_featureID = fID; + m_current = p; +} + +bool RelationTagsBase::IsSkipRelation(std::string const & type) +{ + /// @todo Skip special relation types. + return (type == "multipolygon" || type == "bridge"); +} + +bool RelationTagsBase::IsKeyTagExists(std::string const & key) const +{ + for (auto const & p : m_current->m_tags) + if (p.key == key) + return true; + return false; +} + +void RelationTagsBase::AddCustomTag(pair const & p) +{ + m_current->AddTag(p.first, p.second); +} + +// RelationTagsNode +RelationTagsNode::RelationTagsNode(routing::TagsProcessor & tagsProcessor) : + RelationTagsBase(tagsProcessor) +{ +} + +void RelationTagsNode::Process(RelationElement const & e) +{ + std::string const & type = e.GetType(); + if (TBase::IsSkipRelation(type)) + return; + + if (type == "restriction") + { + m_routingTagsProcessor.m_restrictionWriter.Write(e); + return; + } + + bool const processAssociatedStreet = type == "associatedStreet" && + TBase::IsKeyTagExists("addr:housenumber") && + !TBase::IsKeyTagExists("addr:street"); + for (auto const & p : e.tags) + { + // - used in railway station processing + // - used in routing information + // - used in building addresses matching + if (p.first == "network" || p.first == "operator" || p.first == "route" || + p.first == "maxspeed" || + strings::StartsWith(p.first, "addr:")) + { + if (!TBase::IsKeyTagExists(p.first)) + TBase::AddCustomTag(p); + } + // Convert associatedStreet relation name to addr:street tag if we don't have one. + else if (p.first == "name" && processAssociatedStreet) + TBase::AddCustomTag({"addr:street", p.second}); + } +} + + +// RelationTagsWay +RelationTagsWay::RelationTagsWay(routing::TagsProcessor & routingTagsProcessor) + : RelationTagsBase(routingTagsProcessor) +{ +} + + +bool RelationTagsWay::IsAcceptBoundary(RelationElement const & e) const +{ + std::string role; + CHECK(e.FindWay(TBase::m_featureID, role), (TBase::m_featureID)); + + // Do not accumulate boundary types (boundary=administrative) for inner polygons. + // Example: Minsk city border (admin_level=8) is inner for Minsk area border (admin_level=4). + return (role != "inner"); +} + + +void RelationTagsWay::Process(RelationElement const & e) +{ + /// @todo Review route relations in future. + /// Actually, now they give a lot of dummy tags. + std::string const & type = e.GetType(); + if (TBase::IsSkipRelation(type)) + return; + + if (type == "route") + { + if (e.GetTagValue("route") == "road") + { + // Append "network/ref" to the feature ref tag. + std::string ref = e.GetTagValue("ref"); + if (!ref.empty()) + { + std::string const & network = e.GetTagValue("network"); + // Not processing networks with more than 15 chars (see road_shields_parser.cpp). + if (!network.empty() && network.find('/') == std::string::npos && network.size() < 15) + ref = network + '/' + ref; + std::string const & refBase = m_current->GetTag("ref"); + if (!refBase.empty()) + ref = refBase + ';' + ref; + TBase::AddCustomTag({"ref", std::move(ref)}); + } + } + return; + } + + if (type == "restriction") + { + m_routingTagsProcessor.m_restrictionWriter.Write(e); + return; + } + + if (type == "building") + { + // If this way has "outline" role, add [building=has_parts] type. + if (e.GetWayRole(m_current->id) == "outline") + TBase::AddCustomTag({"building", "has_parts"}); + return; + } + + bool const isBoundary = (type == "boundary") && IsAcceptBoundary(e); + bool const processAssociatedStreet = type == "associatedStreet" && + TBase::IsKeyTagExists("addr:housenumber") && !TBase::IsKeyTagExists("addr:street"); + bool const isHighway = TBase::IsKeyTagExists("highway"); + + for (auto const & p : e.tags) + { + /// @todo Skip common key tags. + if (p.first == "type" || p.first == "route" || p.first == "area") + continue; + + // Convert associatedStreet relation name to addr:street tag if we don't have one. + if (p.first == "name" && processAssociatedStreet) + TBase::AddCustomTag({"addr:street", p.second}); + + // Important! Skip all "name" tags. + if (strings::StartsWith(p.first, "name") || p.first == "int_name") + continue; + + if (!isBoundary && p.first == "boundary") + continue; + + if (p.first == "place") + continue; + + // Do not pass "ref" tags from boundaries and other, non-route relations to highways. + if (p.first == "ref" && isHighway) + continue; + + TBase::AddCustomTag(p); + } +} + +// HolesAccumulator +HolesAccumulator::HolesAccumulator(cache::IntermediateDataReader & holder) : + m_merger(holder) +{ +} + +FeatureBuilder1::Geometry & HolesAccumulator::GetHoles() +{ + ASSERT(m_holes.empty(), ("Can call only once")); + m_merger.ForEachArea(false, [this](FeatureBuilder1::PointSeq const & v, std::vector const &) + { + m_holes.push_back(std::move(v)); + }); + return m_holes; +} + + +// HolesProcessor +HolesProcessor::HolesProcessor(uint64_t id, cache::IntermediateDataReader & holder) + : m_id(id) + , m_holes(holder) +{ +} + +bool HolesProcessor::operator() (uint64_t /*id*/, RelationElement const & e) +{ + if (e.GetType() != "multipolygon") + return false; + std::string role; + if (e.FindWay(m_id, role) && (role == "outer")) + { + e.ForEachWay(*this); + // stop processing (??? assume that "outer way" exists in one relation only ???) + return true; + } + return false; +} + +void HolesProcessor::operator() (uint64_t id, std::string const & role) +{ + if (id != m_id && role == "inner") + m_holes(id); +} + + +// OsmToFeatureTranslator +OsmToFeatureTranslator::OsmToFeatureTranslator(std::shared_ptr emitter, + cache::IntermediateDataReader & holder, + const feature::GenerateInfo & info) : + m_emitter(emitter), + m_holder(holder), + m_coastType(info.m_makeCoasts ? classif().GetCoastType() : 0), + m_nodeRelations(m_routingTagsProcessor), + m_wayRelations(m_routingTagsProcessor), + m_metalinesBuilder(info.GetIntermediateFileName(METALINES_FILENAME, "")) +{ + auto const addrFilePath = info.GetAddressesFileName(); + if (!addrFilePath.empty()) + m_addrWriter.reset(new FileWriter(addrFilePath)); + + auto const restrictionsFilePath = info.GetIntermediateFileName(RESTRICTIONS_FILENAME, ""); + if (!restrictionsFilePath.empty()) + m_routingTagsProcessor.m_restrictionWriter.Open(restrictionsFilePath); + + auto const roadAccessFilePath = info.GetIntermediateFileName(ROAD_ACCESS_FILENAME, ""); + if (!roadAccessFilePath.empty()) + m_routingTagsProcessor.m_roadAccessWriter.Open(roadAccessFilePath); +} + + +void OsmToFeatureTranslator::EmitElement(OsmElement * p) +{ + enum class FeatureState {Unknown, Ok, EmptyTags, HasNoTypes, BrokenRef, ShortGeom, InvalidType}; + + CHECK(p, ("Tried to emit a null OsmElement")); + + FeatureParams params; + FeatureState state = FeatureState::Unknown; + + switch(p->type) + { + case OsmElement::EntityType::Node: + { + if (p->m_tags.empty()) + { + state = FeatureState::EmptyTags; + break; + } + + if (!ParseType(p, params)) + { + state = FeatureState::HasNoTypes; + break; + } + + m2::PointD const pt = MercatorBounds::FromLatLon(p->lat, p->lon); + EmitPoint(pt, params, osm::Id::Node(p->id)); + state = FeatureState::Ok; + break; + } + + case OsmElement::EntityType::Way: + { + FeatureBuilder1 ft; + + // Parse geometry. + for (uint64_t ref : p->Nodes()) + { + m2::PointD pt; + if (!m_holder.GetNode(ref, pt.y, pt.x)) + { + state = FeatureState::BrokenRef; + break; + } + ft.AddPoint(pt); + } + + if (state == FeatureState::BrokenRef) + break; + + if (ft.GetPointsCount() < 2) + { + state = FeatureState::ShortGeom; + break; + } + + if (!ParseType(p, params)) + { + state = FeatureState::HasNoTypes; + break; + } + + ft.SetOsmId(osm::Id::Way(p->id)); + bool isCoastLine = (m_coastType != 0 && params.IsTypeExist(m_coastType)); + + EmitArea(ft, params, [&] (FeatureBuilder1 & ft) + { + isCoastLine = false; // emit coastline feature only once + HolesProcessor processor(p->id, m_holder); + m_holder.ForEachRelationByWay(p->id, processor); + ft.SetAreaAddHoles(processor.GetHoles()); + }); + + m_metalinesBuilder(*p, params); + EmitLine(ft, params, isCoastLine); + state = FeatureState::Ok; + break; + } + + case OsmElement::EntityType::Relation: + { + { + // 1. Check, if this is our processable relation. Here we process only polygon relations. + size_t i = 0; + size_t const count = p->m_tags.size(); + for (; i < count; ++i) + { + if (p->m_tags[i].key == "type" && p->m_tags[i].value == "multipolygon") + break; + } + if (i == count) + { + state = FeatureState::InvalidType; + break; + } + } + + if (!ParseType(p, params)) + { + state = FeatureState::HasNoTypes; + break; + } + + HolesAccumulator holes(m_holder); + AreaWayMerger outer(m_holder); + + // 3. Iterate ways to get 'outer' and 'inner' geometries + for (auto const & e : p->Members()) + { + if (e.type != OsmElement::EntityType::Way) + continue; + + if (e.role == "outer") + outer.AddWay(e.ref); + else if (e.role == "inner") + holes(e.ref); + } + + auto const & holesGeometry = holes.GetHoles(); + outer.ForEachArea(true, [&] (FeatureBuilder1::PointSeq const & pts, std::vector const & ids) + { + FeatureBuilder1 ft; + + for (uint64_t id : ids) + ft.AddOsmId(osm::Id::Way(id)); + + for (auto const & pt : pts) + ft.AddPoint(pt); + + ft.AddOsmId(osm::Id::Relation(p->id)); + EmitArea(ft, params, [&holesGeometry] (FeatureBuilder1 & ft) {ft.SetAreaAddHoles(holesGeometry);}); + }); + + state = FeatureState::Ok; + break; + } + + default: + state = FeatureState::Unknown; + break; + } +} + +bool OsmToFeatureTranslator::ParseType(OsmElement * p, FeatureParams & params) +{ + // Get tags from parent relations. + if (p->type == OsmElement::EntityType::Node) + { + m_nodeRelations.Reset(p->id, p); + m_holder.ForEachRelationByNodeCached(p->id, m_nodeRelations); + } + else if (p->type == OsmElement::EntityType::Way) + { + m_wayRelations.Reset(p->id, p); + m_holder.ForEachRelationByWayCached(p->id, m_wayRelations); + } + + // Get params from element tags. + ftype::GetNameAndType(p, params); + if (!params.IsValid()) + return false; + + m_routingTagsProcessor.m_roadAccessWriter.Process(*p); + return true; +} + +void OsmToFeatureTranslator::EmitPoint(m2::PointD const & pt, + FeatureParams params, osm::Id id) const +{ + if (feature::RemoveNoDrawableTypes(params.m_Types, feature::GEOM_POINT)) + { + FeatureBuilder1 ft; + ft.SetCenter(pt); + ft.SetOsmId(id); + EmitFeatureBase(ft, params); + } +} + +void OsmToFeatureTranslator::EmitLine(FeatureBuilder1 & ft, FeatureParams params, + bool isCoastLine) const +{ + if (isCoastLine || feature::RemoveNoDrawableTypes(params.m_Types, feature::GEOM_LINE)) + { + ft.SetLinear(params.m_reverseGeometry); + EmitFeatureBase(ft, params); + } +} + +void OsmToFeatureTranslator::EmitFeatureBase(FeatureBuilder1 & ft, FeatureParams const & params) const +{ + ft.SetParams(params); + if (ft.PreSerialize()) + { + std::string addr; + if (m_addrWriter && ftypes::IsBuildingChecker::Instance()(params.m_Types) && + ft.FormatFullAddress(addr)) + { + m_addrWriter->Write(addr.c_str(), addr.size()); + } + + (*m_emitter)(ft); + } +} + + +// OsmToFeatureTranslatorRegion +OsmToFeatureTranslatorRegion::OsmToFeatureTranslatorRegion(std::shared_ptr emitter, + cache::IntermediateDataReader & holder) + : m_emitter(emitter) + , m_holder(holder) +{ +} + +void OsmToFeatureTranslatorRegion::EmitElement(OsmElement * p) +{ + CHECK(p, ("Tried to emit a null OsmElement")); + + switch(p->type) + { + case OsmElement::EntityType::Relation: + { + FeatureParams params; + if (!(IsSuitableElement(p) && ParseParams(p, params))) + return; + + BuildFeatureAndEmit(p, params); + break; + } + default: + break; + } +} + +bool OsmToFeatureTranslatorRegion::IsSuitableElement(OsmElement const * p) const +{ + static std::set const adminLevels = {"2", "4", "5", "6", "7", "8"}; + static std::set const places = {"city", "town", "village", "suburb", "neighbourhood", + "hamlet", "locality", "isolated_dwelling"}; + + bool haveBoundary = false; + bool haveAdminLevel = false; + bool haveName = false; + for (auto const & t : p->Tags()) + { if (t.key == "place" && places.find(t.value) != places.end()) + return true; + + if (t.key == "boundary" && t.value == "administrative") + haveBoundary = true; + else if (t.key == "admin_level" && adminLevels.find(t.value) != adminLevels.end()) + haveAdminLevel = true; + else if (t.key == "name" && !t.value.empty()) + haveName = true; + + if (haveBoundary && haveAdminLevel && haveName) + return true; + } + + return false; +} + +void OsmToFeatureTranslatorRegion::AddInfoAboutRegion(OsmElement const * p, + FeatureBuilder1 & ft) const +{ + cout << 1; +} + +bool OsmToFeatureTranslatorRegion::ParseParams(OsmElement * p, FeatureParams & params) const +{ + ftype::GetNameAndType(p, params); + return params.IsValid(); +} + +void OsmToFeatureTranslatorRegion::BuildFeatureAndEmit(OsmElement const * p, FeatureParams & params) +{ + HolesAccumulator holes(m_holder); + AreaWayMerger outer(m_holder); + for (auto const & e : p->Members()) + { + if (e.type != OsmElement::EntityType::Way) + continue; + + if (e.role == "outer") + outer.AddWay(e.ref); + else if (e.role == "inner") + holes(e.ref); + } + + auto const & holesGeometry = holes.GetHoles(); + outer.ForEachArea(true, [&] (FeatureBuilder1::PointSeq const & pts, + std::vector const & ids) + { + FeatureBuilder1 ft; + for (uint64_t id : ids) + ft.AddOsmId(osm::Id::Way(id)); + + for (auto const & pt : pts) + ft.AddPoint(pt); + + ft.AddOsmId(osm::Id::Relation(p->id)); + if (!ft.IsGeometryClosed()) + return; + + ft.SetAreaAddHoles(holesGeometry); + ft.SetParams(params); + if (!ft.PreSerialize()) + return; + + AddInfoAboutRegion(p, ft); + (*m_emitter)(ft); + }); +} + +} // namespace generator diff --git a/generator/osm_translator.hpp b/generator/osm_translator.hpp index 4189e0641e..d23c67ad3b 100644 --- a/generator/osm_translator.hpp +++ b/generator/osm_translator.hpp @@ -1,9 +1,11 @@ #pragma once #include "generator/feature_builder.hpp" +#include "generator/intermediate_data.hpp" #include "generator/metalines_builder.hpp" #include "generator/osm2type.hpp" #include "generator/osm_element.hpp" +#include "generator/osm_source.hpp" #include "generator/restriction_writer.hpp" #include "generator/routing_helpers.hpp" #include "generator/ways_merger.hpp" @@ -26,23 +28,15 @@ #include #include -namespace -{ +namespace generator { + /// Generated features should include parent relation tags to make /// full types matching and storing any additional info. class RelationTagsBase { public: - RelationTagsBase(routing::TagsProcessor & tagsProcessor) - : m_routingTagsProcessor(tagsProcessor), m_cache(14) - { - } - - void Reset(uint64_t fID, OsmElement * p) - { - m_featureID = fID; - m_current = p; - } + explicit RelationTagsBase(routing::TagsProcessor & tagsProcessor); + void Reset(uint64_t fID, OsmElement * p); template bool operator() (uint64_t id, TReader & reader) @@ -57,25 +51,9 @@ public: } protected: - static bool IsSkipRelation(std::string const & type) - { - /// @todo Skip special relation types. - return (type == "multipolygon" || type == "bridge"); - } - - bool IsKeyTagExists(std::string const & key) const - { - for (auto const & p : m_current->m_tags) - if (p.key == key) - return true; - return false; - } - - void AddCustomTag(pair const & p) - { - m_current->AddTag(p.first, p.second); - } - + static bool IsSkipRelation(std::string const & type); + bool IsKeyTagExists(std::string const & key) const; + void AddCustomTag(pair const & p); virtual void Process(RelationElement const & e) = 0; protected: @@ -92,276 +70,83 @@ class RelationTagsNode : public RelationTagsBase using TBase = RelationTagsBase; public: - RelationTagsNode(routing::TagsProcessor & tagsProcessor) : RelationTagsBase(tagsProcessor) {} + explicit RelationTagsNode(routing::TagsProcessor & tagsProcessor); protected: - void Process(RelationElement const & e) override - { - std::string const & type = e.GetType(); - if (TBase::IsSkipRelation(type)) - return; - - if (type == "restriction") - { - m_routingTagsProcessor.m_restrictionWriter.Write(e); - return; - } - - bool const processAssociatedStreet = type == "associatedStreet" && - TBase::IsKeyTagExists("addr:housenumber") && !TBase::IsKeyTagExists("addr:street"); - - for (auto const & p : e.tags) - { - // - used in railway station processing - // - used in routing information - // - used in building addresses matching - if (p.first == "network" || p.first == "operator" || p.first == "route" || - p.first == "maxspeed" || - strings::StartsWith(p.first, "addr:")) - { - if (!TBase::IsKeyTagExists(p.first)) - TBase::AddCustomTag(p); - } - // Convert associatedStreet relation name to addr:street tag if we don't have one. - else if (p.first == "name" && processAssociatedStreet) - TBase::AddCustomTag({"addr:street", p.second}); - } - } + void Process(RelationElement const & e) override; }; + class RelationTagsWay : public RelationTagsBase { public: - RelationTagsWay(routing::TagsProcessor & routingTagsProcessor) - : RelationTagsBase(routingTagsProcessor) - { - } + explicit RelationTagsWay(routing::TagsProcessor & routingTagsProcessor); private: using TBase = RelationTagsBase; using TNameKeys = std::unordered_set; - bool IsAcceptBoundary(RelationElement const & e) const - { - std::string role; - CHECK(e.FindWay(TBase::m_featureID, role), (TBase::m_featureID)); - - // Do not accumulate boundary types (boundary=administrative) for inner polygons. - // Example: Minsk city border (admin_level=8) is inner for Minsk area border (admin_level=4). - return (role != "inner"); - } + bool IsAcceptBoundary(RelationElement const & e) const; protected: - void Process(RelationElement const & e) override - { - /// @todo Review route relations in future. - /// Actually, now they give a lot of dummy tags. - std::string const & type = e.GetType(); - if (TBase::IsSkipRelation(type)) - return; - - if (type == "route") - { - if (e.GetTagValue("route") == "road") - { - // Append "network/ref" to the feature ref tag. - std::string ref = e.GetTagValue("ref"); - if (!ref.empty()) - { - std::string const & network = e.GetTagValue("network"); - // Not processing networks with more than 15 chars (see road_shields_parser.cpp). - if (!network.empty() && network.find('/') == std::string::npos && network.size() < 15) - ref = network + '/' + ref; - std::string const & refBase = m_current->GetTag("ref"); - if (!refBase.empty()) - ref = refBase + ';' + ref; - TBase::AddCustomTag({"ref", std::move(ref)}); - } - } - return; - } - - if (type == "restriction") - { - m_routingTagsProcessor.m_restrictionWriter.Write(e); - return; - } - - if (type == "building") - { - // If this way has "outline" role, add [building=has_parts] type. - if (e.GetWayRole(m_current->id) == "outline") - TBase::AddCustomTag({"building", "has_parts"}); - return; - } - - bool const isBoundary = (type == "boundary") && IsAcceptBoundary(e); - bool const processAssociatedStreet = type == "associatedStreet" && - TBase::IsKeyTagExists("addr:housenumber") && !TBase::IsKeyTagExists("addr:street"); - bool const isHighway = TBase::IsKeyTagExists("highway"); - - for (auto const & p : e.tags) - { - /// @todo Skip common key tags. - if (p.first == "type" || p.first == "route" || p.first == "area") - continue; - - // Convert associatedStreet relation name to addr:street tag if we don't have one. - if (p.first == "name" && processAssociatedStreet) - TBase::AddCustomTag({"addr:street", p.second}); - - // Important! Skip all "name" tags. - if (strings::StartsWith(p.first, "name") || p.first == "int_name") - continue; - - if (!isBoundary && p.first == "boundary") - continue; - - if (p.first == "place") - continue; - - // Do not pass "ref" tags from boundaries and other, non-route relations to highways. - if (p.first == "ref" && isHighway) - continue; - - TBase::AddCustomTag(p); - } - } + void Process(RelationElement const & e) override; }; -} // namespace -/// @param TEmitter Feature accumulating policy -/// @param TCache Nodes, ways, relations holder -template -class OsmToFeatureTranslator + +class HolesAccumulator { - TEmitter & m_emitter; - TCache & m_holder; - uint32_t m_coastType; - unique_ptr m_addrWriter; + AreaWayMerger m_merger; + FeatureBuilder1::Geometry m_holes; - routing::TagsProcessor m_routingTagsProcessor; +public: + explicit HolesAccumulator(cache::IntermediateDataReader & holder); + void operator() (uint64_t id) { m_merger.AddWay(id); } + FeatureBuilder1::Geometry & GetHoles(); +}; - RelationTagsNode m_nodeRelations; - RelationTagsWay m_wayRelations; - feature::MetalinesBuilder m_metalinesBuilder; - class HolesAccumulator - { - AreaWayMerger m_merger; - FeatureBuilder1::Geometry m_holes; +/// Find holes for way with 'id' in first relation. +class HolesProcessor +{ +public: + explicit HolesProcessor(uint64_t id, cache::IntermediateDataReader & holder); + /// 1. relations process function + bool operator() (uint64_t /*id*/, RelationElement const & e); + /// 2. "ways in relation" process function + void operator() (uint64_t id, std::string const & role); + FeatureBuilder1::Geometry & GetHoles() { return m_holes.GetHoles(); } - public: - HolesAccumulator(OsmToFeatureTranslator * pMain) : m_merger(pMain->m_holder) {} +private: + uint64_t m_id; ///< id of way to find it's holes + HolesAccumulator m_holes; +}; - void operator() (uint64_t id) { m_merger.AddWay(id); } - FeatureBuilder1::Geometry & GetHoles() - { - ASSERT(m_holes.empty(), ("Can call only once")); - m_merger.ForEachArea(false, [this](FeatureBuilder1::PointSeq & v, std::vector const &) - { - m_holes.push_back(FeatureBuilder1::PointSeq()); - m_holes.back().swap(v); - }); - return m_holes; - } - }; +class IOsmToFeatureTranslator +{ +public: + virtual ~IOsmToFeatureTranslator() {} + virtual void EmitElement(OsmElement * p) = 0; - /// Find holes for way with 'id' in first relation. - class HolesProcessor - { - uint64_t m_id; ///< id of way to find it's holes - HolesAccumulator m_holes; +}; - public: - HolesProcessor(uint64_t id, OsmToFeatureTranslator * pMain) : m_id(id), m_holes(pMain) {} - /// 1. relations process function - bool operator() (uint64_t /*id*/, RelationElement const & e) - { - if (e.GetType() != "multipolygon") - return false; - std::string role; - if (e.FindWay(m_id, role) && (role == "outer")) - { - e.ForEachWay(*this); - // stop processing (??? assume that "outer way" exists in one relation only ???) - return true; - } - return false; - } +class OsmToFeatureTranslator : public IOsmToFeatureTranslator +{ +public: + explicit OsmToFeatureTranslator(std::shared_ptr emitter, + cache::IntermediateDataReader & holder, + feature::GenerateInfo const & info); - /// 2. "ways in relation" process function - void operator() (uint64_t id, std::string const & role) - { - if (id != m_id && role == "inner") - m_holes(id); - } - - FeatureBuilder1::Geometry & GetHoles() { return m_holes.GetHoles(); } - }; - - bool ParseType(OsmElement * p, FeatureParams & params) - { - // Get tags from parent relations. - if (p->type == OsmElement::EntityType::Node) - { - m_nodeRelations.Reset(p->id, p); - m_holder.ForEachRelationByNodeCached(p->id, m_nodeRelations); - } - else if (p->type == OsmElement::EntityType::Way) - { - m_wayRelations.Reset(p->id, p); - m_holder.ForEachRelationByWayCached(p->id, m_wayRelations); - } - - // Get params from element tags. - ftype::GetNameAndType(p, params); - if (!params.IsValid()) - return false; - - m_routingTagsProcessor.m_roadAccessWriter.Process(*p); - return true; - } - - void EmitFeatureBase(FeatureBuilder1 & ft, FeatureParams const & params) const - { - ft.SetParams(params); - if (ft.PreSerialize()) - { - std::string addr; - if (m_addrWriter && ftypes::IsBuildingChecker::Instance()(params.m_types) && - ft.FormatFullAddress(addr)) - { - m_addrWriter->Write(addr.c_str(), addr.size()); - } - - m_emitter(ft); - } - } + /// The main entry point for parsing process. + void EmitElement(OsmElement * p) override; +private: + bool ParseType(OsmElement * p, FeatureParams & params); + void EmitFeatureBase(FeatureBuilder1 & ft, FeatureParams const & params) const; /// @param[in] params Pass by value because it can be modified. - //@{ - void EmitPoint(m2::PointD const & pt, FeatureParams params, osm::Id id) const - { - if (feature::RemoveNoDrawableTypes(params.m_types, feature::GEOM_POINT)) - { - FeatureBuilder1 ft; - ft.SetCenter(pt); - ft.SetOsmId(id); - EmitFeatureBase(ft, params); - } - } - - void EmitLine(FeatureBuilder1 & ft, FeatureParams params, bool isCoastLine) const - { - if (isCoastLine || feature::RemoveNoDrawableTypes(params.m_types, feature::GEOM_LINE)) - { - ft.SetLinear(params.m_reverseGeometry); - EmitFeatureBase(ft, params); - } - } + void EmitPoint(m2::PointD const & pt, FeatureParams params, osm::Id id) const; + void EmitLine(FeatureBuilder1 & ft, FeatureParams params, bool isCoastLine) const; template void EmitArea(FeatureBuilder1 & ft, FeatureParams params, MakeFnT makeFn) @@ -376,7 +161,7 @@ class OsmToFeatureTranslator { auto fb = ft; makeFn(fb); - m_emitter.EmitCityBoundary(fb, params); + m_emitter->EmitCityBoundary(fb, params); } // Key point here is that IsDrawableLike and RemoveNoDrawableTypes @@ -396,186 +181,36 @@ class OsmToFeatureTranslator EmitPoint(ft.GetGeometryCenter(), params, ft.GetLastOsmId()); } } - //@} -public: - /// The main entry point for parsing process. - void EmitElement(OsmElement * p) - { - enum class FeatureState {Unknown, Ok, EmptyTags, HasNoTypes, BrokenRef, ShortGeom, InvalidType}; +private: + std::shared_ptr m_emitter; + cache::IntermediateDataReader & m_holder; + uint32_t m_coastType; + unique_ptr m_addrWriter; - CHECK(p, ("Tried to emit a null OsmElement")); + routing::TagsProcessor m_routingTagsProcessor; - FeatureParams params; - FeatureState state = FeatureState::Unknown; - - switch(p->type) - { - case OsmElement::EntityType::Node: - { - if (p->m_tags.empty()) - { - state = FeatureState::EmptyTags; - break; - } - - if (!ParseType(p, params)) - { - state = FeatureState::HasNoTypes; - break; - } - - m2::PointD const pt = MercatorBounds::FromLatLon(p->lat, p->lon); - EmitPoint(pt, params, osm::Id::Node(p->id)); - state = FeatureState::Ok; - break; - } - - case OsmElement::EntityType::Way: - { - FeatureBuilder1 ft; - - // Parse geometry. - for (uint64_t ref : p->Nodes()) - { - m2::PointD pt; - if (!m_holder.GetNode(ref, pt.y, pt.x)) - { - state = FeatureState::BrokenRef; - break; - } - ft.AddPoint(pt); - } - - if (state == FeatureState::BrokenRef) - break; - - if (ft.GetPointsCount() < 2) - { - state = FeatureState::ShortGeom; - break; - } - - if (!ParseType(p, params)) - { - state = FeatureState::HasNoTypes; - break; - } - - ft.SetOsmId(osm::Id::Way(p->id)); - bool isCoastLine = (m_coastType != 0 && params.IsTypeExist(m_coastType)); - - EmitArea(ft, params, [&] (FeatureBuilder1 & ft) - { - isCoastLine = false; // emit coastline feature only once - HolesProcessor processor(p->id, this); - m_holder.ForEachRelationByWay(p->id, processor); - ft.SetAreaAddHoles(processor.GetHoles()); - }); - - m_metalinesBuilder(*p, params); - EmitLine(ft, params, isCoastLine); - state = FeatureState::Ok; - break; - } - - case OsmElement::EntityType::Relation: - { - { - // 1. Check, if this is our processable relation. Here we process only polygon relations. - size_t i = 0; - size_t const count = p->m_tags.size(); - for (; i < count; ++i) - { - if (p->m_tags[i].key == "type" && p->m_tags[i].value == "multipolygon") - break; - } - if (i == count) - { - state = FeatureState::InvalidType; - break; - } - } - - if (!ParseType(p, params)) - { - state = FeatureState::HasNoTypes; - break; - } - - HolesAccumulator holes(this); - AreaWayMerger outer(m_holder); - - // 3. Iterate ways to get 'outer' and 'inner' geometries - for (auto const & e : p->Members()) - { - if (e.type != OsmElement::EntityType::Way) - continue; - - if (e.role == "outer") - outer.AddWay(e.ref); - else if (e.role == "inner") - holes(e.ref); - } - - auto const & holesGeometry = holes.GetHoles(); - outer.ForEachArea(true, [&] (FeatureBuilder1::PointSeq const & pts, std::vector const & ids) - { - FeatureBuilder1 ft; - - for (uint64_t id : ids) - ft.AddOsmId(osm::Id::Way(id)); - - for (auto const & pt : pts) - ft.AddPoint(pt); - - ft.AddOsmId(osm::Id::Relation(p->id)); - EmitArea(ft, params, [&holesGeometry] (FeatureBuilder1 & ft) {ft.SetAreaAddHoles(holesGeometry);}); - }); - - state = FeatureState::Ok; - break; - } - - default: - state = FeatureState::Unknown; - break; - } - -// if (state == FeatureState::Ok) -// { -// Classificator const & c = classif(); -// static char const * const stateText[] = {"U", "Ok", "ET", "HNT", "BR", "SG", "IT"}; -// stringstream ss; -// ss << p->id << " [" << stateText[static_cast::type>(state)] << "]"; -// for (auto const & p : params.m_types) -// ss << " " << c.GetReadableObjectName(p); -// ss << endl; -// std::ofstream file("feature_types_new2.txt", ios::app); -// file.write(ss.str().data(), ss.str().size()); -// } - } - -public: - OsmToFeatureTranslator(TEmitter & emitter, TCache & holder, uint32_t coastType, - std::string const & addrFilePath = {}, - std::string const & restrictionsFilePath = {}, - std::string const & roadAccessFilePath = {}, - std::string const & metalinesFilePath = {}) - : m_emitter(emitter) - , m_holder(holder) - , m_coastType(coastType) - , m_nodeRelations(m_routingTagsProcessor) - , m_wayRelations(m_routingTagsProcessor) - , m_metalinesBuilder(metalinesFilePath) - { - if (!addrFilePath.empty()) - m_addrWriter.reset(new FileWriter(addrFilePath)); - - if (!restrictionsFilePath.empty()) - m_routingTagsProcessor.m_restrictionWriter.Open(restrictionsFilePath); - - if (!roadAccessFilePath.empty()) - m_routingTagsProcessor.m_roadAccessWriter.Open(roadAccessFilePath); - } + RelationTagsNode m_nodeRelations; + RelationTagsWay m_wayRelations; + feature::MetalinesBuilder m_metalinesBuilder; }; + +class OsmToFeatureTranslatorRegion : public IOsmToFeatureTranslator +{ +public: + explicit OsmToFeatureTranslatorRegion(std::shared_ptr emitter, + cache::IntermediateDataReader & holder); + + void EmitElement(OsmElement * p) override; +private: + bool IsSuitableElement(OsmElement const * p) const; + void AddInfoAboutRegion(OsmElement const * p, FeatureBuilder1 & ft) const; + bool ParseParams(OsmElement * p, FeatureParams & params) const; + void BuildFeatureAndEmit(OsmElement const * p, FeatureParams & params); + +private: + std::shared_ptr m_emitter; + cache::IntermediateDataReader & m_holder; +}; + +} // namespace generator diff --git a/generator/ways_merger.cpp b/generator/ways_merger.cpp new file mode 100644 index 0000000000..794e7d5cc9 --- /dev/null +++ b/generator/ways_merger.cpp @@ -0,0 +1,20 @@ +#include "generator/ways_merger.hpp" + +namespace generator { + +AreaWayMerger::AreaWayMerger(cache::IntermediateDataReader & holder) : + m_holder(holder) +{ +} + +void AreaWayMerger::AddWay(uint64_t id) +{ + std::shared_ptr e(new WayElement(id)); + if (m_holder.GetWay(id, *e) && e->IsValid()) + { + m_map.insert(make_pair(e->nodes.front(), e)); + m_map.insert(make_pair(e->nodes.back(), e)); + } +} + +} // namespace generator diff --git a/generator/ways_merger.hpp b/generator/ways_merger.hpp index 18bc6d8a04..11af098997 100644 --- a/generator/ways_merger.hpp +++ b/generator/ways_merger.hpp @@ -1,4 +1,6 @@ #pragma once + +#include "generator/intermediate_data.hpp" #include "generator/intermediate_elements.hpp" #include "geometry/point2d.hpp" @@ -7,28 +9,18 @@ #include #include -template +namespace generator { + class AreaWayMerger { using TPointSeq = std::vector; using TWayMap = std::multimap>; using TWayMapIterator = TWayMap::iterator; - THolder & m_holder; - TWayMap m_map; - public: - AreaWayMerger(THolder & holder) : m_holder(holder) {} + explicit AreaWayMerger(cache::IntermediateDataReader & holder); + void AddWay(uint64_t id); - void AddWay(uint64_t id) - { - std::shared_ptr e(new WayElement(id)); - if (m_holder.GetWay(id, *e) && e->IsValid()) - { - m_map.insert(make_pair(e->nodes.front(), e)); - m_map.insert(make_pair(e->nodes.back(), e)); - } - } template void ForEachArea(bool collectID, ToDo toDo) @@ -60,7 +52,7 @@ public: // next 'id' to process id = e->GetOtherEndPoint(id); - pair r = m_map.equal_range(id); + std::pair r = m_map.equal_range(id); // finally erase element 'e' and find next way in chain i = r.second; @@ -80,4 +72,10 @@ public: toDo(points, ids); } } + +private: + cache::IntermediateDataReader & m_holder; + TWayMap m_map; }; + +} // namespace generator