From bdf145b9ef236a97e5a403fc8b4e8f31cf9d732c Mon Sep 17 00:00:00 2001 From: Vladimir Byko-Ianko Date: Tue, 11 Sep 2018 19:42:55 +0300 Subject: [PATCH] Gathering serialization and deserialization of CityRoads in routing. --- generator/city_roads_generator.cpp | 34 +------ routing/CMakeLists.txt | 1 + routing/city_roads_loader.cpp | 17 +--- routing/city_roads_loader.hpp | 27 ------ routing/city_roads_serialization.hpp | 95 +++++++++++++++++++ .../routing/routing.xcodeproj/project.pbxproj | 4 + 6 files changed, 108 insertions(+), 70 deletions(-) create mode 100644 routing/city_roads_serialization.hpp diff --git a/generator/city_roads_generator.cpp b/generator/city_roads_generator.cpp index c3040f1ffc..446b95b50d 100644 --- a/generator/city_roads_generator.cpp +++ b/generator/city_roads_generator.cpp @@ -2,25 +2,20 @@ #include "generator/cities_boundaries_checker.hpp" -#include "routing/city_roads_loader.hpp" +#include "routing/city_roads_serialization.hpp" #include "routing/routing_helpers.hpp" #include "indexer/feature.hpp" #include "indexer/feature_data.cpp" #include "indexer/feature_processor.hpp" -#include "coding/succinct_mapper.hpp" - #include "base/assert.hpp" #include "base/geo_object_id.hpp" -#include #include #include "defines.hpp" -#include "3party/succinct/elias_fano.hpp" - using namespace generator; using namespace std; @@ -75,36 +70,13 @@ void SerializeCityRoads(string const & dataPath, vector && cityRoadFea FilesContainerW cont(dataPath, FileWriter::OP_WRITE_EXISTING); FileWriter w = cont.GetWriter(CITY_ROADS_FILE_TAG); - CityRoadsHeader header; - auto const startOffset = w.Pos(); - header.Serialize(w); - sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end()); - CHECK(adjacent_find(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()) == - cityRoadFeatureIds.cend(), - ("City road feature ids should be unique.")); - succinct::elias_fano::elias_fano_builder builder(cityRoadFeatureIds.back() + 1, - cityRoadFeatureIds.size()); - for (auto fid : cityRoadFeatureIds) - builder.push_back(fid); - - coding::FreezeVisitor visitor(w); - succinct::elias_fano(&builder).map(visitor); - - auto const endOffset = w.Pos(); - header.m_dataSize = static_cast(endOffset - startOffset - sizeof(CityRoadsHeader)); - - w.Seek(startOffset); - header.Serialize(w); - w.Seek(endOffset); - - LOG(LINFO, ("city_roads section is built in", dataPath, ". Serialized", cityRoadFeatureIds.size(), - "road feature ids in cities. Size:", endOffset - startOffset, "bytes.")); + routing::CityRoadsSerializer::Serialize(w, move(cityRoadFeatureIds)); } bool BuildCityRoads(string const & dataPath, OsmIdToBoundariesTable & table) { - LOG(LDEBUG, ("BuildCityRoads(", dataPath, ");")); + LOG(LINFO, ("BuildCityRoads(", dataPath, ");")); vector cityRoadFeatureIds; try { diff --git a/routing/CMakeLists.txt b/routing/CMakeLists.txt index 0014bc0e99..453615ff39 100644 --- a/routing/CMakeLists.txt +++ b/routing/CMakeLists.txt @@ -22,6 +22,7 @@ set( checkpoints.hpp city_roads_loader.cpp city_roads_loader.hpp + city_roads_serialization.hpp coding.hpp cross_mwm_connector.cpp cross_mwm_connector.hpp diff --git a/routing/city_roads_loader.cpp b/routing/city_roads_loader.cpp index 8868281795..f77bfae3f6 100644 --- a/routing/city_roads_loader.cpp +++ b/routing/city_roads_loader.cpp @@ -1,9 +1,10 @@ #include "routing/city_roads_loader.hpp" +#include "routing/city_roads_serialization.hpp" + #include "indexer/data_source.hpp" #include "coding/reader.hpp" -#include "coding/succinct_mapper.hpp" #include "base/logging.hpp" @@ -23,17 +24,9 @@ CityRoadsLoader::CityRoadsLoader(DataSource const & dataSource, MwmSet::MwmId co try { - auto reader = std::make_unique(mwmValue.m_cont.GetReader(CITY_ROADS_FILE_TAG)); - ReaderSource src(*reader); - - CityRoadsHeader header; - header.Deserialize(src); - - std::vector data(header.m_dataSize); - src.Read(data.data(), data.size()); - m_cityRoadsRegion = std::make_unique(std::move(data)); - coding::MapVisitor visitor(m_cityRoadsRegion->ImmutableData()); - m_cityRoads.map(visitor); + FilesContainerR::TReader reader(mwmValue.m_cont.GetReader(CITY_ROADS_FILE_TAG)); + ReaderSource src(reader); + CityRoadsSerializer::Deserialize(src, m_cityRoadsRegion, m_cityRoads); } catch (Reader::OpenException const & e) { diff --git a/routing/city_roads_loader.hpp b/routing/city_roads_loader.hpp index 8ce92dbd40..b95da89c09 100644 --- a/routing/city_roads_loader.hpp +++ b/routing/city_roads_loader.hpp @@ -3,7 +3,6 @@ #include "indexer/mwm_set.hpp" #include "coding/memory_region.hpp" -#include "coding/writer.hpp" #include #include @@ -15,32 +14,6 @@ class DataSource; namespace routing { -struct CityRoadsHeader -{ - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, m_version); - WriteToSink(sink, m_endianness); - WriteToSink(sink, m_dataSize); - } - - template - void Deserialize(Source & src) - { - m_version = ReadPrimitiveFromSource(src); - m_endianness = ReadPrimitiveFromSource(src); - m_dataSize = ReadPrimitiveFromSource(src); - } - - uint16_t m_version = 0; - // Field |m_endianness| is reserved for endianness of the section. - uint16_t m_endianness = 0; - uint32_t m_dataSize = 0; -}; - -static_assert(sizeof(CityRoadsHeader) == 8, "Wrong header size of city_roads section."); - class CityRoadsLoader { public: diff --git a/routing/city_roads_serialization.hpp b/routing/city_roads_serialization.hpp new file mode 100644 index 0000000000..994e49b870 --- /dev/null +++ b/routing/city_roads_serialization.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include "routing/city_roads_loader.hpp" + +#include "coding/succinct_mapper.hpp" +#include "coding/writer.hpp" + +#include "base/assert.hpp" +#include "base/logging.hpp" + +#include +#include +#include +#include + +#include "3party/succinct/elias_fano.hpp" + +namespace routing +{ +struct CityRoadsHeader +{ + template + void Serialize(Sink & sink) const + { + WriteToSink(sink, m_version); + WriteToSink(sink, m_endianness); + WriteToSink(sink, m_dataSize); + } + + template + void Deserialize(Source & src) + { + m_version = ReadPrimitiveFromSource(src); + m_endianness = ReadPrimitiveFromSource(src); + m_dataSize = ReadPrimitiveFromSource(src); + } + + uint16_t m_version = 0; + // Field |m_endianness| is reserved for endianness of the section. + uint16_t m_endianness = 0; + uint32_t m_dataSize = 0; +}; + +static_assert(sizeof(CityRoadsHeader) == 8, "Wrong header size of city_roads section."); + +class CityRoadsSerializer +{ +public: + CityRoadsSerializer() = delete; + + template + static void Serialize(Sink & sink, std::vector && cityRoadFeatureIds) + { + CityRoadsHeader header; + auto const startOffset = sink.Pos(); + header.Serialize(sink); + + std::sort(cityRoadFeatureIds.begin(), cityRoadFeatureIds.end()); + CHECK(adjacent_find(cityRoadFeatureIds.cbegin(), cityRoadFeatureIds.cend()) == + cityRoadFeatureIds.cend(), + ("City road feature ids should be unique.")); + succinct::elias_fano::elias_fano_builder builder(cityRoadFeatureIds.back() + 1, + cityRoadFeatureIds.size()); + for (auto fid : cityRoadFeatureIds) + builder.push_back(fid); + + coding::FreezeVisitor visitor(sink); + succinct::elias_fano(&builder).map(visitor); + + auto const endOffset = sink.Pos(); + header.m_dataSize = static_cast(endOffset - startOffset - sizeof(CityRoadsHeader)); + + sink.Seek(startOffset); + header.Serialize(sink); + sink.Seek(endOffset); + + LOG(LINFO, ("Serialized", cityRoadFeatureIds.size(), + "road feature ids in cities. Size:", endOffset - startOffset, "bytes.")); + } + + template + static void Deserialize(Source & src, std::unique_ptr & cityRoadsRegion, + succinct::elias_fano & cityRoads) + { + CityRoadsHeader header; + header.Deserialize(src); + + std::vector data(header.m_dataSize); + src.Read(data.data(), data.size()); + cityRoadsRegion = std::make_unique(std::move(data)); + coding::MapVisitor visitor(cityRoadsRegion->ImmutableData()); + cityRoads.map(visitor); + } +}; +} // namespace routing diff --git a/xcode/routing/routing.xcodeproj/project.pbxproj b/xcode/routing/routing.xcodeproj/project.pbxproj index 7bee841833..2856995a81 100644 --- a/xcode/routing/routing.xcodeproj/project.pbxproj +++ b/xcode/routing/routing.xcodeproj/project.pbxproj @@ -91,6 +91,7 @@ 56290B87206A3232003892E0 /* routing_algorithm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 56290B85206A3231003892E0 /* routing_algorithm.cpp */; }; 56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56290B86206A3231003892E0 /* routing_algorithm.hpp */; }; 562BDE2020D14860008EFF6F /* routing_callbacks.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */; }; + 56473D74214825B6007E6FBA /* city_roads_serialization.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 56473D72214825B4007E6FBA /* city_roads_serialization.hpp */; }; 56555E561D897C90009D786D /* libalohalitics.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACE61C68A23B009CB89E /* libalohalitics.a */; }; 56555E581D897C9D009D786D /* liboauthcpp.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6742ACFA1C68A2D7009CB89E /* liboauthcpp.a */; }; 56555E591D897D28009D786D /* testingmain.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6742ACDE1C68A13F009CB89E /* testingmain.cpp */; }; @@ -373,6 +374,7 @@ 56290B85206A3231003892E0 /* routing_algorithm.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = routing_algorithm.cpp; sourceTree = ""; }; 56290B86206A3231003892E0 /* routing_algorithm.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_algorithm.hpp; sourceTree = ""; }; 562BDE1F20D14860008EFF6F /* routing_callbacks.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = routing_callbacks.hpp; sourceTree = ""; }; + 56473D72214825B4007E6FBA /* city_roads_serialization.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = city_roads_serialization.hpp; sourceTree = ""; }; 5661A5CD20DE51C500C6B1D1 /* tools.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = tools.hpp; sourceTree = ""; }; 567059591F3AF96D0062672D /* checkpoint_predictor_test.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor_test.cpp; sourceTree = ""; }; 5670595B1F3AF97F0062672D /* checkpoint_predictor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = checkpoint_predictor.cpp; sourceTree = ""; }; @@ -785,6 +787,7 @@ 675343FA1A3F640D00A0A8C3 /* routing */ = { isa = PBXGroup; children = ( + 56473D72214825B4007E6FBA /* city_roads_serialization.hpp */, 56085C3A21413221004159B4 /* city_roads_loader.cpp */, 56085C3B21413222004159B4 /* city_roads_loader.hpp */, 44E5574B2136EED000B01439 /* speed_camera_ser_des.cpp */, @@ -996,6 +999,7 @@ 0C0DF9221DE898B70055A22F /* index_graph_starter.hpp in Headers */, 56290B88206A3232003892E0 /* routing_algorithm.hpp in Headers */, 40A111CE1F2F6776005E6AD5 /* route_weight.hpp in Headers */, + 56473D74214825B6007E6FBA /* city_roads_serialization.hpp in Headers */, 0C090C881E4E276700D52AFD /* world_graph.hpp in Headers */, 0C08AA351DF83223004195DD /* index_graph_serialization.hpp in Headers */, 5694CECD1EBA25F7004576D3 /* road_access.hpp in Headers */,