From e86680646f078572655373c9591d294f952aba28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BE=D0=B1=D1=80=D1=8B=D0=B8=CC=86=20=D0=AD=D1=8D?= =?UTF-8?q?=D1=85?= Date: Tue, 29 Nov 2016 23:26:21 +0300 Subject: [PATCH] index graph serialiser: first compressed version --- generator/routing_index_generator.cpp | 13 ++- indexer/indexer.pro | 1 - indexer/routing_section.hpp | 32 ------ routing/index_graph.cpp | 8 +- routing/index_graph.hpp | 29 ++--- routing/index_graph_serializer.cpp | 7 ++ routing/index_graph_serializer.hpp | 154 ++++++++++++++++++++++++++ routing/road_index.cpp | 7 ++ routing/road_index.hpp | 65 +++-------- routing/routing.pro | 2 + routing/single_mwm_router.cpp | 6 +- 11 files changed, 212 insertions(+), 112 deletions(-) delete mode 100644 indexer/routing_section.hpp create mode 100644 routing/index_graph_serializer.cpp create mode 100644 routing/index_graph_serializer.hpp diff --git a/generator/routing_index_generator.cpp b/generator/routing_index_generator.cpp index 078f16eb13..e806f74e00 100644 --- a/generator/routing_index_generator.cpp +++ b/generator/routing_index_generator.cpp @@ -1,12 +1,12 @@ #include "generator/routing_index_generator.hpp" #include "routing/index_graph.hpp" +#include "routing/index_graph_serializer.hpp" #include "routing/routing_helpers.hpp" #include "indexer/feature.hpp" #include "indexer/feature_processor.hpp" #include "indexer/point_to_int64.hpp" -#include "indexer/routing_section.hpp" #include "coding/file_container.hpp" @@ -74,15 +74,16 @@ bool BuildRoutingIndex(string const & filename) IndexGraph graph; processor.BuildGraph(graph); - LOG(LINFO, ("Routing index contains", graph.GetNumRoads(), "roads,", graph.GetNumJoints(), - "joints,", graph.GetNumPoints(), "points")); FilesContainerW cont(filename, FileWriter::OP_WRITE_EXISTING); FileWriter writer = cont.GetWriter(ROUTING_FILE_TAG); - RoutingSectionHeader const routingHeader; - routingHeader.Serialize(writer); - graph.Serialize(writer); + auto const startPos = writer.Pos(); + IndexGraphSerializer::Serialize(graph, writer); + auto const sectionSize = writer.Pos() - startPos; + + LOG(LINFO, ("Routing section created:", sectionSize, "bytes,", graph.GetNumRoads(), "roads,", + graph.GetNumJoints(), "joints,", graph.GetNumPoints(), "points")); return true; } catch (RootException const & e) diff --git a/indexer/indexer.pro b/indexer/indexer.pro index 9b68c979fb..4eddc95282 100644 --- a/indexer/indexer.pro +++ b/indexer/indexer.pro @@ -119,7 +119,6 @@ HEADERS += \ point_to_int64.hpp \ postcodes_matcher.hpp \ # it's in indexer due to editor wich is in indexer and depends on postcodes_marcher rank_table.hpp \ - routing_section.hpp \ scale_index.hpp \ scale_index_builder.hpp \ scales.hpp \ diff --git a/indexer/routing_section.hpp b/indexer/routing_section.hpp deleted file mode 100644 index 89bcfc764c..0000000000 --- a/indexer/routing_section.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include "coding/reader.hpp" -#include "coding/write_to_sink.hpp" - -#include "std/cstdint.hpp" - -namespace feature -{ -class RoutingSectionHeader final -{ -public: - RoutingSectionHeader() : m_version(0) {} - - uint8_t GetVersion() const { return m_version; } - - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, m_version); - } - - template - void Deserialize(Source & src) - { - m_version = ReadPrimitiveFromSource(src); - } - -private: - uint8_t m_version; -}; -} // namespace feature diff --git a/routing/index_graph.cpp b/routing/index_graph.cpp index 2a5fde1693..f54378b16e 100644 --- a/routing/index_graph.cpp +++ b/routing/index_graph.cpp @@ -23,10 +23,12 @@ m2::PointD const & IndexGraph::GetPoint(Joint::Id jointId) return m_geometry.GetPoint(m_jointIndex.GetPoint(jointId)); } +void IndexGraph::Build(uint32_t numJoints) { m_jointIndex.Build(m_roadIndex, numJoints); } + void IndexGraph::Import(vector const & joints) { m_roadIndex.Import(joints); - m_jointIndex.Build(m_roadIndex, joints.size()); + Build(joints.size()); } Joint::Id IndexGraph::InsertJoint(RoadPoint const & rp) @@ -66,8 +68,8 @@ void IndexGraph::GetNeighboringEdges(RoadPoint const & rp, bool isOutgoing, GetNeighboringEdge(road, rp, true /* forward */, edges); } -void IndexGraph::GetNeighboringEdge(RoadGeometry const & road, RoadPoint const & rp, - bool forward, vector & edges) const +void IndexGraph::GetNeighboringEdge(RoadGeometry const & road, RoadPoint const & rp, bool forward, + vector & edges) const { pair const & neighbor = m_roadIndex.FindNeighbor(rp, forward); if (neighbor.first != Joint::kInvalidId) diff --git a/routing/index_graph.hpp b/routing/index_graph.hpp index f6e6cf0379..1235fac5f3 100644 --- a/routing/index_graph.hpp +++ b/routing/index_graph.hpp @@ -7,9 +7,6 @@ #include "routing/road_index.hpp" #include "routing/road_point.hpp" -#include "coding/reader.hpp" -#include "coding/write_to_sink.hpp" - #include "geometry/point2d.hpp" #include "std/cstdint.hpp" @@ -55,31 +52,27 @@ public: uint32_t GetNumJoints() const { return m_jointIndex.GetNumJoints(); } uint32_t GetNumPoints() const { return m_jointIndex.GetNumPoints(); } + void Build(uint32_t numJoints); void Import(vector const & joints); + RoadJointIds & InitRoad(uint32_t featureId, uint32_t maxPointId) + { + return m_roadIndex.InitRoad(featureId, maxPointId); + } Joint::Id InsertJoint(RoadPoint const & rp); bool JointLiesOnRoad(Joint::Id jointId, uint32_t featureId) const; + template + void ForEachRoad(F && f) const + { + m_roadIndex.ForEachRoad(forward(f)); + } + template void ForEachPoint(Joint::Id jointId, F && f) const { m_jointIndex.ForEachPoint(jointId, forward(f)); } - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, static_cast(GetNumJoints())); - m_roadIndex.Serialize(sink); - } - - template - void Deserialize(Source & src) - { - uint32_t const jointsSize = ReadPrimitiveFromSource(src); - m_roadIndex.Deserialize(src); - m_jointIndex.Build(m_roadIndex, jointsSize); - } - private: void GetNeighboringEdge(RoadGeometry const & road, RoadPoint const & rp, bool forward, vector & edges) const; diff --git a/routing/index_graph_serializer.cpp b/routing/index_graph_serializer.cpp new file mode 100644 index 0000000000..7401a0c483 --- /dev/null +++ b/routing/index_graph_serializer.cpp @@ -0,0 +1,7 @@ +#include "routing/index_graph_serializer.hpp" + +namespace routing +{ +// static +uint8_t constexpr IndexGraphSerializer::kVersion; +} // namespace routing diff --git a/routing/index_graph_serializer.hpp b/routing/index_graph_serializer.hpp new file mode 100644 index 0000000000..4f031f3c2d --- /dev/null +++ b/routing/index_graph_serializer.hpp @@ -0,0 +1,154 @@ +#pragma once + +#include "routing/index_graph.hpp" +#include "routing/routing_exception.hpp" + +#include "coding/bit_streams.hpp" +#include "coding/elias_coder.hpp" +#include "coding/reader.hpp" +#include "coding/write_to_sink.hpp" + +#include "base/bits.hpp" + +#include "std/cstdint.hpp" +#include "std/limits.hpp" + +namespace routing +{ +class IndexGraphSerializer final +{ +public: + IndexGraphSerializer() = delete; + + template + static void Serialize(IndexGraph const & graph, Sink & sink) + { + WriteToSink(sink, kVersion); + + uint32_t const numJoints = graph.GetNumJoints(); + WriteToSink(sink, numJoints); + uint32_t const bitsPerJoint = bits::NumUsedBits(numJoints); + + map roads; + graph.ForEachRoad( + [&](uint32_t featureId, RoadJointIds const & road) { roads[featureId] = road; }); + + WriteToSink(sink, static_cast(roads.size())); + + BitWriter writer(sink); + + // -1 for uint32_t is some confusing, but it allows process first iteration in the common way. + // It works because uint32_t is modular ring type. + uint32_t prevFeatureId = -1; + + for (auto const & it : roads) + { + uint32_t const featureId = it.first; + RoadJointIds const & road = it.second; + + uint32_t const featureDelta = featureId - prevFeatureId; + if (!Encode32(writer, featureDelta)) + { + MYTHROW(RoutingException, ("Can't encode featureDelta =", featureDelta, ", prevFeatureId =", + prevFeatureId, ", featureId =", featureId)); + } + + prevFeatureId = featureId; + + uint32_t const pointCap = road.GetMaxPointId() + 1; + if (!Encode32(writer, pointCap)) + { + MYTHROW(RoutingException, + ("Can't encode pointCap =", pointCap, ", featureId =", featureId)); + } + + uint32_t prevPointId = -1; + road.ForEachJoint([&](uint32_t pointId, Joint::Id jointId) { + uint32_t const pointDelta = pointId - prevPointId; + if (!Encode32(writer, pointDelta)) + { + MYTHROW(RoutingException, + ("Can't encode pointDelta =", pointDelta, ", prevPointId =", prevPointId, + ", pointId =", pointId, ", featureId =", featureId)); + } + + prevPointId = pointId; + writer.WriteAtMost32Bits(jointId, bitsPerJoint); + }); + } + } + + template + static void Deserialize(IndexGraph & graph, Source & src) + { + uint8_t const version = ReadPrimitiveFromSource(src); + if (version != kVersion) + { + MYTHROW(RoutingException, + ("Unknown index graph version =", version, ", current version =", kVersion)); + } + + uint32_t const numJoints = ReadPrimitiveFromSource(src); + uint32_t const bitsPerJoint = bits::NumUsedBits(numJoints); + uint32_t const numRoads = ReadPrimitiveFromSource(src); + + BitReader reader(src); + + uint32_t featureId = -1; + for (uint32_t i = 0; i < numRoads; ++i) + { + uint32_t const featureDelta = Decode32(reader); + featureId += featureDelta; + + uint32_t const pointCap = Decode32(reader); + if (pointCap < 1) + MYTHROW(RoutingException, ("Invalid pointCap =", pointCap, ", featureId =", featureId)); + + uint32_t const maxPointId = pointCap - 1; + + RoadJointIds & roadJoints = graph.InitRoad(featureId, maxPointId); + + for (uint32_t pointId = -1; pointId != maxPointId;) + { + uint32_t const pointDelta = Decode32(reader); + pointId += pointDelta; + if (pointId > maxPointId) + { + MYTHROW(RoutingException, ("Invalid pointId =", pointId, ", maxPointId =", maxPointId, + ", pointDelta =", pointDelta, ", featureId =", featureId)); + } + + Joint::Id const jointId = reader.ReadAtMost32Bits(bitsPerJoint); + if (jointId >= numJoints) + { + MYTHROW(RoutingException, ("Invalid jointId =", jointId, ", numJoints =", numJoints, + ", pointId =", pointId, ", featureId =", featureId)); + } + + roadJoints.AddJoint(pointId, jointId); + } + } + + graph.Build(numJoints); + } + +private: + static uint8_t constexpr kVersion = 0; + + template + static bool Encode32(BitWriter & writer, uint32_t value) + { + return coding::GammaCoder::Encode(writer, static_cast(value)); + } + + template + static uint32_t Decode32(BitReader & reader) + { + uint64_t const decoded = coding::GammaCoder::Decode(reader); + if (decoded > numeric_limits::max()) + MYTHROW(RoutingException, ("Invalid uint32_t decoded", decoded)); + + return static_cast(decoded); + } +}; +} // namespace routing diff --git a/routing/road_index.cpp b/routing/road_index.cpp index 6f9d925fe7..5023ad6890 100644 --- a/routing/road_index.cpp +++ b/routing/road_index.cpp @@ -18,6 +18,13 @@ void RoadIndex::Import(vector const & joints) } } +RoadJointIds & RoadIndex::InitRoad(uint32_t featureId, uint32_t maxPointId) +{ + RoadJointIds & road = m_roads[featureId]; + road.Init(maxPointId); + return road; +} + pair RoadIndex::FindNeighbor(RoadPoint const & rp, bool forward) const { auto const it = m_roads.find(rp.GetFeatureId()); diff --git a/routing/road_index.hpp b/routing/road_index.hpp index f35fca1066..a3ee532f1c 100644 --- a/routing/road_index.hpp +++ b/routing/road_index.hpp @@ -2,9 +2,6 @@ #include "routing/joint.hpp" -#include "coding/reader.hpp" -#include "coding/write_to_sink.hpp" - #include "std/algorithm.hpp" #include "std/cstdint.hpp" #include "std/unordered_map.hpp" @@ -16,6 +13,12 @@ namespace routing class RoadJointIds final { public: + void Init(uint32_t maxPointId) + { + m_jointIds.clear(); + m_jointIds.reserve(maxPointId + 1); + } + Joint::Id GetJointId(uint32_t pointId) const { if (pointId < m_jointIds.size()) @@ -26,6 +29,8 @@ public: void AddJoint(uint32_t pointId, Joint::Id jointId) { + ASSERT_NOT_EQUAL(jointId, Joint::kInvalidId, ()); + if (pointId >= m_jointIds.size()) m_jointIds.insert(m_jointIds.end(), pointId + 1 - m_jointIds.size(), Joint::kInvalidId); @@ -33,6 +38,13 @@ public: m_jointIds[pointId] = jointId; } + uint32_t GetMaxPointId() const + { + ASSERT(!m_jointIds.empty(), ()); + ASSERT_NOT_EQUAL(m_jointIds.back(), Joint::kInvalidId, ()); + return static_cast(m_jointIds.size() - 1); + } + template void ForEachJoint(F && f) const { @@ -77,27 +89,6 @@ public: return result; } - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, static_cast(m_jointIds.size())); - for (Joint::Id jointId : m_jointIds) - WriteToSink(sink, jointId); - } - - template - void Deserialize(Source & src) - { - m_jointIds.clear(); - Joint::Id const jointsSize = ReadPrimitiveFromSource(src); - m_jointIds.reserve(jointsSize); - for (Joint::Id i = 0; i < jointsSize; ++i) - { - Joint::Id const jointId = ReadPrimitiveFromSource(src); - m_jointIds.emplace_back(jointId); - } - } - private: // Joint ids indexed by point id. // If some point id doesn't match any joint id, this vector contains Joint::kInvalidId. @@ -114,35 +105,13 @@ public: m_roads[rp.GetFeatureId()].AddJoint(rp.GetPointId(), jointId); } + RoadJointIds & InitRoad(uint32_t featureId, uint32_t maxPointId); + // Find nearest point with normal joint id. // If forward == true: neighbor with larger point id (right neighbor) // If forward == false: neighbor with smaller point id (left neighbor) pair FindNeighbor(RoadPoint const & rp, bool forward) const; - template - void Serialize(Sink & sink) const - { - WriteToSink(sink, static_cast(m_roads.size())); - for (auto const & it : m_roads) - { - uint32_t const featureId = it.first; - WriteToSink(sink, featureId); - it.second.Serialize(sink); - } - } - - template - void Deserialize(Source & src) - { - m_roads.clear(); - size_t const roadsSize = static_cast(ReadPrimitiveFromSource(src)); - for (size_t i = 0; i < roadsSize; ++i) - { - uint32_t featureId = ReadPrimitiveFromSource(src); - m_roads[featureId].Deserialize(src); - } - } - uint32_t GetSize() const { return m_roads.size(); } Joint::Id GetJointId(RoadPoint const & rp) const diff --git a/routing/routing.pro b/routing/routing.pro index cff8f507b6..92f9b1f2d3 100644 --- a/routing/routing.pro +++ b/routing/routing.pro @@ -27,6 +27,7 @@ SOURCES += \ features_road_graph.cpp \ geometry.cpp \ index_graph.cpp \ + index_graph_serializer.cpp \ index_graph_starter.cpp \ joint.cpp \ joint_index.cpp \ @@ -77,6 +78,7 @@ HEADERS += \ features_road_graph.hpp \ geometry.hpp \ index_graph.hpp \ + index_graph_serializer.hpp \ index_graph_starter.hpp \ joint.hpp \ joint_index.hpp \ diff --git a/routing/single_mwm_router.cpp b/routing/single_mwm_router.cpp index 89648b59b6..0c51547074 100644 --- a/routing/single_mwm_router.cpp +++ b/routing/single_mwm_router.cpp @@ -6,6 +6,7 @@ #include "routing/bicycle_model.hpp" #include "routing/car_model.hpp" #include "routing/index_graph.hpp" +#include "routing/index_graph_serializer.hpp" #include "routing/index_graph_starter.hpp" #include "routing/pedestrian_model.hpp" #include "routing/route.hpp" @@ -13,7 +14,6 @@ #include "routing/turns_generator.hpp" #include "indexer/feature_altitude.hpp" -#include "indexer/routing_section.hpp" #include "geometry/distance.hpp" #include "geometry/mercator.hpp" @@ -196,9 +196,7 @@ bool SingleMwmRouter::LoadIndex(MwmSet::MwmId const & mwmId, string const & coun my::Timer timer; FilesContainerR::TReader reader(mwmValue->m_cont.GetReader(ROUTING_FILE_TAG)); ReaderSource src(reader); - feature::RoutingSectionHeader header; - header.Deserialize(src); - graph.Deserialize(src); + IndexGraphSerializer::Deserialize(graph, src); LOG(LINFO, (ROUTING_FILE_TAG, "section for", country, "loaded in", timer.ElapsedSeconds(), "seconds")); return true;