From 486ed6a098bf978063e82898c02b122826c8e5ba Mon Sep 17 00:00:00 2001 From: Sergey Yershov Date: Tue, 4 Aug 2015 16:47:03 +0300 Subject: [PATCH] Rename and refactor in generator --- generator/data_cache_file.hpp | 18 +- generator/generate_info.hpp | 77 +- generator/generator_tool/generator_tool.cpp | 34 +- generator/osm_source.cpp | 746 +++++++++----------- generator/osm_source.hpp | 7 +- 5 files changed, 444 insertions(+), 438 deletions(-) diff --git a/generator/data_cache_file.hpp b/generator/data_cache_file.hpp index 57ffbf81bd..e40eb0501b 100644 --- a/generator/data_cache_file.hpp +++ b/generator/data_cache_file.hpp @@ -54,7 +54,7 @@ public: string GetFileName() const { return m_file.GetName(); } - void Flush() + void WriteAll() { if (m_elements.empty()) return; @@ -89,10 +89,10 @@ public: LOG_SHORT(LINFO, ("Offsets reading is finished")); } - void Write(TKey k, TValue const & v) + void Add(TKey k, TValue const & v) { if (m_elements.size() > kFlushCount) - Flush(); + WriteAll(); m_elements.push_back(make_pair(k, v)); } @@ -145,11 +145,11 @@ public: template void Write(TKey id, TValue const & value) { - m_offsets.Write(id, m_stream.Pos()); + m_offsets.Add(id, m_stream.Pos()); m_stream << value; } - void SaveOffsets() { m_offsets.Flush(); } + void SaveOffsets() { m_offsets.WriteAll(); } }; class DataFileReader : public DataFileBase @@ -193,16 +193,16 @@ protected: TData m_ways; TData m_relations; - TIndex m_nodes2rel; - TIndex m_ways2rel; + TIndex m_nodeToRelations; + TIndex m_wayToRelations; public: BaseFileHolder(TNodesHolder & nodes, string const & dir) : m_nodes(nodes) , m_ways(my::JoinFoldersToPath(dir, WAYS_FILE)) , m_relations(my::JoinFoldersToPath(dir, RELATIONS_FILE)) - , m_nodes2rel(my::JoinFoldersToPath(dir, string(NODES_FILE) + ID2REL_EXT)) - , m_ways2rel(my::JoinFoldersToPath(dir, string(WAYS_FILE) + ID2REL_EXT)) + , m_nodeToRelations(my::JoinFoldersToPath(dir, string(NODES_FILE) + ID2REL_EXT)) + , m_wayToRelations(my::JoinFoldersToPath(dir, string(WAYS_FILE) + ID2REL_EXT)) { } }; diff --git a/generator/generate_info.hpp b/generator/generate_info.hpp index 0797b50362..9a0ebf6e60 100644 --- a/generator/generate_info.hpp +++ b/generator/generate_info.hpp @@ -2,6 +2,8 @@ #include "defines.hpp" +#include "base/logging.hpp" + #include "coding/file_name_utils.hpp" #include "std/string.hpp" @@ -12,12 +14,70 @@ namespace feature struct GenerateInfo { + enum class NodeStorageType + { + Memory, + Index, + File + }; + + enum class OsmSourceType + { + XML, + O5M + }; + + + // Directory for .mwm.tmp files. + string m_tmpDir; + // Directory for result .mwm files. + string m_targetDir; + // Directory for all intermediate files. + string m_intermediateDir; + + // Current generated file name if --output option is defined. + string m_fileName; + + NodeStorageType m_nodeStorageType; + OsmSourceType m_osmFileType; + string m_osmFileName; + + vector m_bucketNames; + + bool m_createWorld; + bool m_splitByPolygons; + bool m_makeCoasts, m_emitCoasts; + bool m_genAddresses; + + GenerateInfo() : m_createWorld(false), m_splitByPolygons(false), m_makeCoasts(false), m_emitCoasts(false), m_genAddresses(false) { } + void SetOsmFileType(string const & type) + { + if (type == "xml") + m_osmFileType = OsmSourceType::XML; + else if (type == "o5m") + m_osmFileType = OsmSourceType::O5M; + else + LOG(LCRITICAL, ("Unknown source type:", type)); + } + + void SetNodeStorageType(string const & type) + { + if (type == "raw") + m_nodeStorageType = NodeStorageType::File; + else if (type == "map") + m_nodeStorageType = NodeStorageType::Index; + else if (type == "mem") + m_nodeStorageType = NodeStorageType::Memory; + else + LOG(LCRITICAL, ("Incorrect node_storage type:", type)); + } + string GetTmpFileName(string const & fileName, char const * ext = DATA_FILE_EXTENSION_TMP) const { return my::JoinFoldersToPath(m_tmpDir, fileName + ext); @@ -34,23 +94,6 @@ struct GenerateInfo { return ((m_genAddresses && !m_fileName.empty()) ? GetTargetFileName(m_fileName, ADDR_FILE_EXTENSION) : string()); } - - // Directory for .mwm.tmp files. - string m_tmpDir; - // Directory for result .mwm files. - string m_targetDir; - // Directory for all intermediate files. - string m_intermediateDir; - - // Current generated file name if --output option is defined. - string m_fileName; - - vector m_bucketNames; - - bool m_createWorld; - bool m_splitByPolygons; - bool m_makeCoasts, m_emitCoasts; - bool m_genAddresses; }; } // namespace feature diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 5b10b4b60a..99f7fa1631 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -85,21 +85,10 @@ int main(int argc, char ** argv) string const path = FLAGS_data_path.empty() ? pl.WritableDir() : my::AddSlashIfNeeded(FLAGS_data_path); - // Generating intermediate files - if (FLAGS_preprocess) - { - LOG(LINFO, ("Generating intermediate data ....")); - if (!GenerateIntermediateData(FLAGS_intermediate_data_path, FLAGS_node_storage, - FLAGS_osm_file_type, FLAGS_osm_file_name)) - { - return -1; - } - } feature::GenerateInfo genInfo; - genInfo.m_intermediateDir = FLAGS_intermediate_data_path.empty() - ? path - : my::AddSlashIfNeeded(FLAGS_intermediate_data_path); + genInfo.m_intermediateDir = FLAGS_intermediate_data_path.empty() ? path + : my::AddSlashIfNeeded(FLAGS_intermediate_data_path); genInfo.m_targetDir = genInfo.m_tmpDir = path; /// @todo Probably, it's better to add separate option for .mwm.tmp files. @@ -110,6 +99,23 @@ int main(int argc, char ** argv) genInfo.m_tmpDir = tmpPath; } + genInfo.m_osmFileName = FLAGS_osm_file_name; + + if (!FLAGS_node_storage.empty()) + genInfo.SetNodeStorageType(FLAGS_node_storage); + if (!FLAGS_osm_file_type.empty()) + genInfo.SetOsmFileType(FLAGS_osm_file_type); + + // Generating intermediate files + if (FLAGS_preprocess) + { + LOG(LINFO, ("Generating intermediate data ....")); + if (!GenerateIntermediateData(genInfo)) + { + return -1; + } + } + // load classificator only if necessary if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry || FLAGS_generate_index || FLAGS_generate_search_index || @@ -132,7 +138,7 @@ int main(int argc, char ** argv) genInfo.m_fileName = FLAGS_output; genInfo.m_genAddresses = FLAGS_generate_addresses_file; - if (!GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name)) + if (!GenerateFeatures(genInfo)) return -1; if (FLAGS_generate_world) diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index d3564ed99b..3a7c9100ae 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -4,13 +4,13 @@ #include "generator/first_pass_parser.hpp" #include "generator/osm_decl.hpp" #include "generator/osm_element.hpp" +#include "generator/osm_o5m_source.hpp" #include "generator/osm_source.hpp" #include "generator/point_storage.hpp" #include "generator/polygonizer.hpp" #include "generator/source_reader.hpp" #include "generator/world_map_generator.hpp" #include "generator/xml_element.hpp" -#include "generator/osm_o5m_source.hpp" #include "indexer/classificator.hpp" #include "indexer/mercator.hpp" @@ -19,322 +19,299 @@ #include "defines.hpp" - #define DECODE_O5M_COORD(coord) (static_cast(coord) / 1E+7) -namespace feature -{ - - template - class FileHolder : public cache::BaseFileHolder - { - using TReader = cache::DataFileReader; - using TBase = cache::BaseFileHolder; - - using typename TBase::TKey; - - template - struct ElementProcessorBase - { - protected: - TReader & m_reader; - ToDo & m_toDo; - - public: - ElementProcessorBase(TReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} - - bool operator()(uint64_t id) - { - TElement e; - return m_reader.Read(id, e) ? m_toDo(id, e) : false; - } - }; - - template - struct RelationProcessor : public ElementProcessorBase - { - using TBase = ElementProcessorBase; - - RelationProcessor(TReader & reader, ToDo & toDo) : TBase(reader, toDo) {} - }; - - template - struct CachedRelationProcessor : public RelationProcessor - { - using TBase = RelationProcessor; - - CachedRelationProcessor(TReader & rels, ToDo & toDo) : TBase(rels, toDo) {} - bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } - }; - - public: - FileHolder(TNodesHolder & holder, string const & dir) : TBase(holder, dir) {} - - bool GetNode(uint64_t id, double & lat, double & lng) - { - return this->m_nodes.GetPoint(id, lat, lng); - } - - bool GetWay(TKey id, WayElement & e) - { - return this->m_ways.Read(id, e); - } - - template - void ForEachRelationByWay(TKey id, ToDo && toDo) - { - RelationProcessor processor(this->m_relations, toDo); - this->m_ways2rel.ForEachByKey(id, processor); - } - - template - void ForEachRelationByNodeCached(TKey id, ToDo && toDo) - { - CachedRelationProcessor processor(this->m_relations, toDo); - this->m_nodes2rel.ForEachByKey(id, processor); - } - - template - void ForEachRelationByWayCached(TKey id, ToDo && toDo) - { - CachedRelationProcessor processor(this->m_relations, toDo); - this->m_ways2rel.ForEachByKey(id, processor); - } - - void LoadIndex() - { - this->m_ways.LoadOffsets(); - this->m_relations.LoadOffsets(); - - this->m_nodes2rel.ReadAll(); - this->m_ways2rel.ReadAll(); - } - }; -} // namespace feature - -namespace data -{ - - template - class FileHolder : public cache::BaseFileHolder - { - typedef cache::BaseFileHolder base_type; - - typedef typename base_type::TKey user_id_t; - - template - void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v) - { - for (size_t i = 0; i < v.size(); ++i) - rMap.Write(v[i].first, relid); - } - - public: - FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {} - - void AddNode(uint64_t id, double lat, double lng) - { - this->m_nodes.AddPoint(id, lat, lng); - } - - void AddWay(user_id_t id, WayElement const & e) - { - this->m_ways.Write(id, e); - } - - void AddRelation(user_id_t id, RelationElement const & e) - { - const string relationType = e.GetType(); - if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary") - { - this->m_relations.Write(id, e); - - add_id2rel_vector(this->m_nodes2rel, id, e.nodes); - add_id2rel_vector(this->m_ways2rel, id, e.ways); - } - } - - void SaveIndex() - { - this->m_ways.SaveOffsets(); - this->m_relations.SaveOffsets(); - - this->m_nodes2rel.Flush(); - this->m_ways2rel.Flush(); - } - }; -} // namespace data - namespace { - class MainFeaturesEmitter +template +class IntermediateDataReader + : public cache::BaseFileHolder +{ + using TReader = cache::DataFileReader; + using TBase = cache::BaseFileHolder; + + using typename TBase::TKey; + using TBase::m_nodes; + using TBase::m_nodeToRelations; + using TBase::m_ways; + using TBase::m_wayToRelations; + using TBase::m_relations; + + template + struct ElementProcessorBase { - typedef WorldMapGenerator WorldGenerator; - typedef CountryMapGenerator > CountriesGenerator; - unique_ptr m_countries; - unique_ptr m_world; - - unique_ptr m_coasts; - unique_ptr m_coastsHolder; - - string m_srcCoastsFile; - - template class CombinedEmitter - { - T1 * m_p1; - T2 * m_p2; - public: - CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {} - void operator() (FeatureBuilder1 const & fb, uint64_t) - { - if (m_p1) (*m_p1)(fb); - if (m_p2) (*m_p2)(fb); - } - }; - - enum TypeIndex - { - NATURAL_COASTLINE, - NATURAL_LAND, - PLACE_ISLAND, - PLACE_ISLET, - - TYPES_COUNT - }; - uint32_t m_types[TYPES_COUNT]; - - inline uint32_t Type(TypeIndex i) const { return m_types[i]; } + protected: + TReader & m_reader; + ToDo & m_toDo; public: - MainFeaturesEmitter(feature::GenerateInfo const & info) + ElementProcessorBase(TReader & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} + + bool operator()(uint64_t id) { - Classificator const & c = classif(); - - char const * arr[][2] = { - { "natural", "coastline" }, - { "natural", "land" }, - { "place", "island" }, - { "place", "islet" } - }; - static_assert(ARRAY_SIZE(arr) == TYPES_COUNT, ""); - - for (size_t i = 0; i < ARRAY_SIZE(arr); ++i) - m_types[i] = c.GetTypeByPath(vector(arr[i], arr[i] + 2)); - - m_srcCoastsFile = info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, ".geom"); - - CHECK(!info.m_makeCoasts || !info.m_createWorld, - ("We can't do make_coasts and generate_world at the same time")); - - if (info.m_makeCoasts) - { - m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE))); - - m_coastsHolder.reset(new feature::FeaturesAndRawGeometryCollector( - m_srcCoastsFile, info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION))); - return; - } - - if (info.m_emitCoasts) - m_coastsHolder.reset(new feature::FeaturesCollector(info.GetTmpFileName(WORLD_COASTS_FILE_NAME))); - - if (info.m_splitByPolygons || !info.m_fileName.empty()) - m_countries.reset(new CountriesGenerator(info)); - - if (info.m_createWorld) - m_world.reset(new WorldGenerator(info)); - } - - void operator() (FeatureBuilder1 fb) - { - uint32_t const coastType = Type(NATURAL_COASTLINE); - bool const hasCoast = fb.HasType(coastType); - - if (m_coasts) - { - if (hasCoast) - { - CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () ); - - // leave only coastline type - fb.SetType(coastType); - (*m_coasts)(fb); - } - } - else - { - if (hasCoast) - { - fb.PopExactType(Type(NATURAL_LAND)); - fb.PopExactType(coastType); - } - else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) && - fb.GetGeomType() == feature::GEOM_AREA) - { - fb.AddType(Type(NATURAL_LAND)); - } - - if (fb.RemoveInvalidTypes()) - { - if (m_world) - (*m_world)(fb); - - if (m_countries) - (*m_countries)(fb); - } - } - } - - /// @return false if coasts are not merged and FLAG_fail_on_coasts is set - bool Finish() - { - if (m_world) - m_world->DoMerge(); - - if (m_coasts) - { - // Check and stop if some coasts were not merged - if (!m_coasts->Finish()) - return false; - - LOG(LINFO, ("Generating coastline polygons")); - - size_t totalFeatures = 0; - size_t totalPoints = 0; - size_t totalPolygons = 0; - - vector vecFb; - m_coasts->GetFeatures(vecFb); - - for (auto & fb : vecFb) - { - (*m_coastsHolder)(fb); - - ++totalFeatures; - totalPoints += fb.GetPointsCount(); - totalPolygons += fb.GetPolygonsCount(); - } - LOG(LINFO, ("Total features:", totalFeatures, "total polygons:", totalPolygons, "total points:", totalPoints)); - } - else if (m_coastsHolder) - { - CombinedEmitter - emitter(m_coastsHolder.get(), m_countries.get()); - feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter); - } - return true; - } - - inline void GetNames(vector & names) const - { - if (m_countries) - names = m_countries->Parent().Names(); - else - names.clear(); + TElement e; + return m_reader.Read(id, e) ? m_toDo(id, e) : false; } }; -} // namespace anonymous + template + struct RelationProcessor : public ElementProcessorBase + { + using TBase = ElementProcessorBase; + + RelationProcessor(TReader & reader, ToDo & toDo) : TBase(reader, toDo) {} + }; + + template + struct CachedRelationProcessor : public RelationProcessor + { + using TBase = RelationProcessor; + + CachedRelationProcessor(TReader & rels, ToDo & toDo) : TBase(rels, toDo) {} + bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } + }; + +public: + IntermediateDataReader(TNodesHolder & holder, string const & dir) : TBase(holder, dir) {} + + bool GetNode(TKey id, double & lat, double & lng) { return m_nodes.GetPoint(id, lat, lng); } + + bool GetWay(TKey id, WayElement & e) { return m_ways.Read(id, e); } + + template + void ForEachRelationByWay(TKey id, ToDo && toDo) + { + RelationProcessor processor(m_relations, toDo); + m_wayToRelations.ForEachByKey(id, processor); + } + + template + void ForEachRelationByNodeCached(TKey id, ToDo && toDo) + { + CachedRelationProcessor processor(m_relations, toDo); + m_nodeToRelations.ForEachByKey(id, processor); + } + + template + void ForEachRelationByWayCached(TKey id, ToDo && toDo) + { + CachedRelationProcessor processor(m_relations, toDo); + m_wayToRelations.ForEachByKey(id, processor); + } + + void LoadIndex() + { + m_ways.LoadOffsets(); + m_relations.LoadOffsets(); + + m_nodeToRelations.ReadAll(); + m_wayToRelations.ReadAll(); + } +}; + +template +class IntermediateDataWriter + : public cache::BaseFileHolder +{ + using TBase = cache::BaseFileHolder; + + using typename TBase::TKey; + using TBase::m_nodes; + using TBase::m_nodeToRelations; + using TBase::m_ways; + using TBase::m_wayToRelations; + using TBase::m_relations; + + template + static void AddToIndex(TIndex & index, TKey relationId, TContainer const & values) + { + for (auto const & v : values) + index.Add(v.first, relationId); + } + +public: + IntermediateDataWriter(TNodesHolder & nodes, string const & dir) : TBase(nodes, dir) {} + + void AddNode(TKey id, double lat, double lng) { m_nodes.AddPoint(id, lat, lng); } + + void AddWay(TKey id, WayElement const & e) { m_ways.Write(id, e); } + + void AddRelation(TKey id, RelationElement const & e) + { + string const & relationType = e.GetType(); + if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary")) + return; + + m_relations.Write(id, e); + AddToIndex(m_nodeToRelations, id, e.nodes); + AddToIndex(m_wayToRelations, id, e.ways); + } + + void SaveIndex() + { + m_ways.SaveOffsets(); + m_relations.SaveOffsets(); + + m_nodeToRelations.WriteAll(); + m_wayToRelations.WriteAll(); + } +}; + +class MainFeaturesEmitter +{ + using TWorldGenerator = WorldMapGenerator; + using TCountriesGenerator = CountryMapGenerator>; + unique_ptr m_countries; + unique_ptr m_world; + + unique_ptr m_coasts; + unique_ptr m_coastsHolder; + + string m_srcCoastsFile; + + enum TypeIndex + { + NATURAL_COASTLINE, + NATURAL_LAND, + PLACE_ISLAND, + PLACE_ISLET, + + TYPES_COUNT + }; + uint32_t m_types[TYPES_COUNT]; + + inline uint32_t Type(TypeIndex i) const { return m_types[i]; } + +public: + MainFeaturesEmitter(feature::GenerateInfo const & info) + { + Classificator const & c = classif(); + + char const * arr[][2] = { + {"natural", "coastline"}, {"natural", "land"}, {"place", "island"}, {"place", "islet"}}; + static_assert(ARRAY_SIZE(arr) == TYPES_COUNT, ""); + + for (size_t i = 0; i < ARRAY_SIZE(arr); ++i) + m_types[i] = c.GetTypeByPath({arr[i][0], arr[i][1]}); + + m_srcCoastsFile = info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, ".geom"); + + CHECK(!info.m_makeCoasts || !info.m_createWorld, + ("We can't do make_coasts and generate_world at the same time")); + + if (info.m_makeCoasts) + { + m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE))); + + m_coastsHolder.reset(new feature::FeaturesAndRawGeometryCollector( + m_srcCoastsFile, + info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION))); + return; + } + + if (info.m_emitCoasts) + m_coastsHolder.reset( + new feature::FeaturesCollector(info.GetTmpFileName(WORLD_COASTS_FILE_NAME))); + + if (info.m_splitByPolygons || !info.m_fileName.empty()) + m_countries.reset(new TCountriesGenerator(info)); + + if (info.m_createWorld) + m_world.reset(new TWorldGenerator(info)); + } + + void operator()(FeatureBuilder1 fb) + { + uint32_t const coastType = Type(NATURAL_COASTLINE); + bool const hasCoast = fb.HasType(coastType); + + if (m_coasts) + { + if (hasCoast) + { + CHECK(fb.GetGeomType() != feature::GEOM_POINT, ()); + // leave only coastline type + fb.SetType(coastType); + (*m_coasts)(fb); + } + return; + } + + if (hasCoast) + { + fb.PopExactType(Type(NATURAL_LAND)); + fb.PopExactType(coastType); + } + else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) && + fb.GetGeomType() == feature::GEOM_AREA) + { + fb.AddType(Type(NATURAL_LAND)); + } + + if (!fb.RemoveInvalidTypes()) + return; + + if (m_world) + (*m_world)(fb); + + if (m_countries) + (*m_countries)(fb); + } + + /// @return false if coasts are not merged and FLAG_fail_on_coasts is set + bool Finish() + { + if (m_world) + m_world->DoMerge(); + + if (m_coasts) + { + // Check and stop if some coasts were not merged + if (!m_coasts->Finish()) + return false; + + LOG(LINFO, ("Generating coastline polygons")); + + size_t totalFeatures = 0; + size_t totalPoints = 0; + size_t totalPolygons = 0; + + vector vecFb; + m_coasts->GetFeatures(vecFb); + + for (auto & fb : vecFb) + { + (*m_coastsHolder)(fb); + + ++totalFeatures; + totalPoints += fb.GetPointsCount(); + totalPolygons += fb.GetPolygonsCount(); + } + LOG(LINFO, ("Total features:", totalFeatures, "total polygons:", totalPolygons, + "total points:", totalPoints)); + } + else if (m_coastsHolder) + { + feature::ForEachFromDatRawFormat(m_srcCoastsFile, [this](FeatureBuilder1 const & fb, uint64_t) + { + if (m_coastsHolder) + (*m_coastsHolder)(fb); + if (m_countries) + (*m_countries)(fb); + }); + } + return true; + } + + inline void GetNames(vector & names) const + { + if (m_countries) + names = m_countries->Parent().Names(); + else + names.clear(); + } +}; +} // anonymous namespace template void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder) @@ -343,7 +320,7 @@ void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder) osm::O5MSource dataset([&stream](uint8_t * buffer, size_t size) { - return stream.Read(reinterpret_cast(buffer), size); + return stream.Read(reinterpret_cast(buffer), size); }); for (auto const & em : dataset) @@ -352,11 +329,11 @@ void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder) { case TType::Node: { - // Could do something with em.id, em.lon, em.lat here, lon and lat are ints in 1E+7 * degree units + // Could do something with em.id, em.lon, em.lat here + // lon and lat are ints in 1E+7 * degree units // convert to mercator - double const lat = MercatorBounds::LatToY(DECODE_O5M_COORD(em.lat)); - double const lng = MercatorBounds::LonToX(DECODE_O5M_COORD(em.lon)); - holder.AddNode(em.id, lat, lng); + auto const pt = MercatorBounds::FromLatLon(DECODE_O5M_COORD(em.lat), DECODE_O5M_COORD(em.lon)); + holder.AddNode(em.id, pt.y, pt.x); break; } case TType::Way: @@ -376,7 +353,8 @@ void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder) RelationElement relation; for (auto const & member : em.Members()) { - // Could do something with member ref (way or node or rel id depends on type), type and role + // Could do something with member ref (way or node or rel id depends on type), type and + // role if (member.type == TType::Node) relation.nodes.emplace_back(make_pair(member.ref, string(member.role))); else if (member.type == TType::Way) @@ -404,7 +382,7 @@ void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser) osm::O5MSource dataset([&stream](uint8_t * buffer, size_t size) { - return stream.Read(reinterpret_cast(buffer), size); + return stream.Read(reinterpret_cast(buffer), size); }); for (auto const & em : dataset) @@ -433,28 +411,24 @@ void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser) p.tagKey = XMLElement::ET_RELATION; for (auto const & member : em.Members()) { - string strType; switch (member.type) { case TType::Node: - strType = "node"; + p.AddMEMBER(member.ref, "node", member.role); break; case TType::Way: - strType = "way"; + p.AddMEMBER(member.ref, "way", member.role); break; case TType::Relation: - strType = "relation"; + p.AddMEMBER(member.ref, "relation", member.role); break; - default: - break; + default: break; } - p.AddMEMBER(member.ref, strType, member.role); } break; } - default: - break; + default: break; } for (auto const & tag : em.Tags()) @@ -464,46 +438,38 @@ void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser) } } - /////////////////////////////////////////////////////////////////////////////////////////////////// // Generate functions implementations. /////////////////////////////////////////////////////////////////////////////////////////////////// - -template -bool GenerateFeaturesImpl(feature::GenerateInfo & info, string const &osmFileType, string const & osmFileName = string()) +template +bool GenerateFeaturesImpl(feature::GenerateInfo & info) { try { - NodesHolderT nodes(info.m_intermediateDir + NODES_FILE); + TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE)); - typedef feature::FileHolder HolderT; - HolderT holder(nodes, info.m_intermediateDir); - holder.LoadIndex(); + using TDataCache = IntermediateDataReader; + TDataCache cache(nodes, info.m_intermediateDir); + cache.LoadIndex(); MainFeaturesEmitter bucketer(info); - SecondPassParser parser( - bucketer, holder, - info.m_makeCoasts ? classif().GetCoastType() : 0, - info.GetAddressesFileName()); + SecondPassParser parser( + bucketer, cache, info.m_makeCoasts ? classif().GetCoastType() : 0, + info.GetAddressesFileName()); - SourceReader reader(osmFileName); - - if (osmFileType == "xml") + SourceReader reader(info.m_osmFileName); + switch (info.m_osmFileType) { - ParseXMLSequence(reader, parser); - } - else if (osmFileType == "o5m") - { - BuildFeaturesFromO5M(reader, parser); - } - else - { - LOG(LERROR, ("Unknown source type:", osmFileType)); - return false; + case feature::GenerateInfo::OsmSourceType::XML: + ParseXMLSequence(reader, parser); + break; + case feature::GenerateInfo::OsmSourceType::O5M: + BuildFeaturesFromO5M(reader, parser); + break; } - LOG(LINFO, ("Processing", osmFileType, "file done.")); + LOG(LINFO, ("Processing", info.m_osmFileName, "done.")); parser.Finish(); @@ -515,78 +481,72 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info, string const &osmFileTyp } catch (Reader::Exception const & e) { - LOG(LERROR, ("Error with file ", e.what())); - return false; + LOG(LCRITICAL, ("Error with file ", e.what())); } return true; } template -bool GenerateIntermediateDataImpl(string const & dir, string const &osmFileType, string const & osmFileName = string()) +bool GenerateIntermediateDataImpl(feature::GenerateInfo & info) { try { - TNodesHolder nodes(dir + NODES_FILE); - typedef data::FileHolder HolderT; - HolderT holder(nodes, dir); + TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE)); + using TDataCache = IntermediateDataWriter; + TDataCache cache(nodes, info.m_intermediateDir); - SourceReader reader(osmFileName); + SourceReader reader(info.m_osmFileName); - LOG(LINFO, ("Data sorce format:", osmFileType)); + LOG(LINFO, ("Data source:", info.m_osmFileName)); - if (osmFileType == "xml") + switch (info.m_osmFileType) { - FirstPassParser parser(holder); - ParseXMLSequence(reader, parser); - } - else if (osmFileType == "o5m") - { - BuildIntermediateDataFromO5M(reader, holder); - } - else - { - LOG(LERROR, ("Unknown source type:", osmFileType)); - return false; + case feature::GenerateInfo::OsmSourceType::XML: + { + FirstPassParser parser(cache); + ParseXMLSequence(reader, parser); + break; + } + case feature::GenerateInfo::OsmSourceType::O5M: + BuildIntermediateDataFromO5M(reader, cache); + break; } + cache.SaveIndex(); LOG(LINFO, ("Added points count = ", nodes.GetCount())); - - holder.SaveIndex(); } catch (Writer::Exception const & e) { - LOG(LERROR, ("Error with file ", e.what())); - return false; + LOG(LCRITICAL, ("Error with file ", e.what())); } return true; } - -bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName) +bool GenerateFeatures(feature::GenerateInfo & info) { - if (nodeStorage == "raw") - return GenerateFeaturesImpl>(info, osmFileType, osmFileName); - else if (nodeStorage == "map") - return GenerateFeaturesImpl>(info, osmFileType, osmFileName); - else if (nodeStorage == "mem") - return GenerateFeaturesImpl>(info, osmFileType, osmFileName); - else - CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); + switch (info.m_nodeStorageType) + { + case feature::GenerateInfo::NodeStorageType::File: + return GenerateFeaturesImpl>(info); + case feature::GenerateInfo::NodeStorageType::Index: + return GenerateFeaturesImpl>(info); + case feature::GenerateInfo::NodeStorageType::Memory: + return GenerateFeaturesImpl>(info); + } return false; } - -bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName) +bool GenerateIntermediateData(feature::GenerateInfo & info) { - if (nodeStorage == "raw") - return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); - else if (nodeStorage == "map") - return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); - else if (nodeStorage == "mem") - return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); - else - CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); + switch (info.m_nodeStorageType) + { + case feature::GenerateInfo::NodeStorageType::File: + return GenerateIntermediateDataImpl>(info); + case feature::GenerateInfo::NodeStorageType::Index: + return GenerateIntermediateDataImpl>(info); + case feature::GenerateInfo::NodeStorageType::Memory: + return GenerateIntermediateDataImpl>(info); + } return false; } - diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp index ba195e5ab0..e4ad42c197 100644 --- a/generator/osm_source.hpp +++ b/generator/osm_source.hpp @@ -2,10 +2,7 @@ #include "generator/generate_info.hpp" -#include "std/string.hpp" - - -bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName); -bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName); +bool GenerateFeatures(feature::GenerateInfo & info); +bool GenerateIntermediateData(feature::GenerateInfo & info);