From fa78f5fc87582dfd5553d1fc8e4a2b5e16574185 Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Thu, 19 Feb 2015 16:39:47 +0300 Subject: [PATCH] User defined index for feature to routing request --- defines.hpp | 1 + generator/routing_generator.cpp | 2 +- routing/osrm2feature_map.cpp | 55 ++++----- routing/osrm2feature_map.hpp | 131 +++++++++++++++++++-- routing/osrm_router.cpp | 30 ++--- routing/osrm_router.hpp | 4 +- routing/routing_tests/osrm_router_test.cpp | 4 +- 7 files changed, 167 insertions(+), 60 deletions(-) diff --git a/defines.hpp b/defines.hpp index 8fec4e0fdb..e350fd97ca 100644 --- a/defines.hpp +++ b/defines.hpp @@ -25,6 +25,7 @@ #define ROUTING_FTSEG_FILE_TAG "ftseg" #define ROUTING_NODEIND_TO_FTSEGIND_FILE_TAG "node2ftseg" +#define FTSEG_MAPPING_BACKWARD_INDEX "bftseg" //Secret word to unlock experimental features in production builds #define ROUTING_SECRET_UNLOCKING_WORD "?ariadna" diff --git a/generator/routing_generator.cpp b/generator/routing_generator.cpp index 2baaf6ebe1..160183b9bb 100644 --- a/generator/routing_generator.cpp +++ b/generator/routing_generator.cpp @@ -225,7 +225,7 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin ++found; // Emit segment. - OsrmFtSegMapping::FtSeg ftSeg(fID, ind1, ind2); + OsrmMappingTypes::FtSeg ftSeg(fID, ind1, ind2); if (vec.empty() || !vec.back().Merge(ftSeg)) { vec.push_back(ftSeg); diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp index deb1ad09ae..1daac2b8b7 100644 --- a/routing/osrm2feature_map.cpp +++ b/routing/osrm2feature_map.cpp @@ -20,7 +20,7 @@ namespace routing OsrmNodeIdT const INVALID_NODE_ID = -1; -OsrmFtSegMapping::FtSeg::FtSeg(uint32_t fid, uint32_t ps, uint32_t pe) +OsrmMappingTypes::FtSeg::FtSeg(uint32_t fid, uint32_t ps, uint32_t pe) : m_fid(fid), m_pointStart(static_cast(ps)), m_pointEnd(static_cast(pe)) @@ -30,17 +30,17 @@ OsrmFtSegMapping::FtSeg::FtSeg(uint32_t fid, uint32_t ps, uint32_t pe) CHECK_EQUAL(m_pointEnd, pe, ()); } -OsrmFtSegMapping::FtSeg::FtSeg(uint64_t x) +OsrmMappingTypes::FtSeg::FtSeg(uint64_t x) : m_fid(x & 0xFFFFFFFF), m_pointStart(x >> 48), m_pointEnd((x >> 32) & 0xFFFF) { } -uint64_t OsrmFtSegMapping::FtSeg::Store() const +uint64_t OsrmMappingTypes::FtSeg::Store() const { return (uint64_t(m_pointStart) << 48) + (uint64_t(m_pointEnd) << 32) + uint64_t(m_fid); } -bool OsrmFtSegMapping::FtSeg::Merge(FtSeg const & other) +bool OsrmMappingTypes::FtSeg::Merge(FtSeg const & other) { if (other.m_fid != m_fid) return false; @@ -67,7 +67,7 @@ bool OsrmFtSegMapping::FtSeg::Merge(FtSeg const & other) return false; } -bool OsrmFtSegMapping::FtSeg::IsIntersect(FtSeg const & other) const +bool OsrmMappingTypes::FtSeg::IsIntersect(FtSeg const & other) const { if (other.m_fid != m_fid) return false; @@ -80,7 +80,7 @@ bool OsrmFtSegMapping::FtSeg::IsIntersect(FtSeg const & other) const return my::IsIntersect(s1, e1, s2, e2); } -string DebugPrint(OsrmFtSegMapping::FtSeg const & seg) +string OsrmMappingTypes::DebugPrint(OsrmMappingTypes::FtSeg const & seg) { stringstream ss; ss << "{ fID = " << seg.m_fid << @@ -89,7 +89,7 @@ string DebugPrint(OsrmFtSegMapping::FtSeg const & seg) return ss.str(); } -string DebugPrint(OsrmFtSegMapping::SegOffset const & off) +string OsrmMappingTypes::DebugPrint(OsrmMappingTypes::SegOffset const & off) { stringstream ss; ss << "{ " << off.m_nodeId << ", " << off.m_offset << " }"; @@ -126,6 +126,7 @@ void OsrmFtSegMapping::Map(FilesMappingContainer & cont) m_handle.Assign(cont.Map(ROUTING_FTSEG_FILE_TAG)); ASSERT(m_handle.IsValid(), ()); succinct::mapper::map(m_segments, m_handle.GetData()); + m_backwardIndex.Construct(m_segments, cont); } void OsrmFtSegMapping::Unmap() @@ -143,7 +144,7 @@ void OsrmFtSegMapping::DumpSegmentsByFID(uint32_t fID) const #ifdef DEBUG for (size_t i = 0; i < m_segments.size(); ++i) { - FtSeg s(m_segments[i]); + OsrmMappingTypes::FtSeg s(m_segments[i]); if (s.m_fid == fID) LOG(LDEBUG, (s)); } @@ -153,7 +154,7 @@ void OsrmFtSegMapping::DumpSegmentsByFID(uint32_t fID) const void OsrmFtSegMapping::DumpSegmentByNode(OsrmNodeIdT nodeId) const { #ifdef DEBUG - ForEachFtSeg(nodeId, [] (FtSeg const & s) + ForEachFtSeg(nodeId, [] (OsrmMappingTypes::FtSeg const & s) { LOG(LDEBUG, (s)); }); @@ -186,19 +187,17 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola return true; }; - size_t const count = GetSegmentsCount(); - for (size_t i = 0; i < count && !segments.empty(); ++i) + for (auto it = segments.begin(); it != segments.end(); ++it) { - FtSeg s(m_segments[i]); + OsrmMappingTypes::FtSeg const & seg = *(*it); + vector results; + m_backwardIndex.GetIndexesByFid(seg.m_fid, results); - if (requestCancel) - return; - - for (auto it = segments.begin(); it != segments.end(); ++it) + for (size_t i : results) { - FtSeg const & seg = *(*it); - if (s.m_fid != seg.m_fid) - continue; + OsrmMappingTypes::FtSeg const s(m_segments[i]); + + ASSERT(s.m_fid == seg.m_fid, ()); if (s.m_pointStart <= s.m_pointEnd) { @@ -206,7 +205,6 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola { if (addResFn(seg.Store(), i, true)) { - segments.erase(it); break; } } @@ -217,7 +215,6 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola { if (addResFn(seg.Store(), i, false)) { - segments.erase(it); break; } } @@ -226,16 +223,16 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola } } -void OsrmFtSegMapping::GetSegmentByIndex(size_t idx, FtSeg & seg) const +void OsrmFtSegMapping::GetSegmentByIndex(size_t idx, OsrmMappingTypes::FtSeg & seg) const { ASSERT_LESS(idx, m_segments.size(), ()); - FtSeg(m_segments[idx]).Swap(seg); + OsrmMappingTypes::FtSeg(m_segments[idx]).Swap(seg); } pair OsrmFtSegMapping::GetSegmentsRange(OsrmNodeIdT nodeId) const { - SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), SegOffset(nodeId, 0), - [] (SegOffset const & o, SegOffset const & val) + SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), OsrmMappingTypes::SegOffset(nodeId, 0), + [] (OsrmMappingTypes::SegOffset const & o, OsrmMappingTypes::SegOffset const & val) { return (o.m_nodeId < val.m_nodeId); }); @@ -251,8 +248,8 @@ pair OsrmFtSegMapping::GetSegmentsRange(OsrmNodeIdT nodeId) cons OsrmNodeIdT OsrmFtSegMapping::GetNodeId(size_t segInd) const { - SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), SegOffset(segInd, 0), - [] (SegOffset const & o, SegOffset const & val) + SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), OsrmMappingTypes::SegOffset(segInd, 0), + [] (OsrmMappingTypes::SegOffset const & o, OsrmMappingTypes::SegOffset const & val) { return (o.m_nodeId + o.m_offset < val.m_nodeId); }); @@ -280,7 +277,7 @@ void OsrmFtSegMappingBuilder::Append(OsrmNodeIdT nodeId, FtSegVectorT const & da size_t const count = data.size(); if (count == 0) - m_buffer.push_back(FtSeg(FtSeg::INVALID_FID, 0, 1).Store()); + m_buffer.push_back(OsrmMappingTypes::FtSeg(OsrmMappingTypes::FtSeg::INVALID_FID, 0, 1).Store()); else { for (size_t i = 0; i < count; ++i) @@ -294,7 +291,7 @@ void OsrmFtSegMappingBuilder::Append(OsrmNodeIdT nodeId, FtSegVectorT const & da uint32_t const off = static_cast(m_lastOffset); CHECK_EQUAL(m_lastOffset, off, ()); - m_offsets.push_back(SegOffset(nodeId, off)); + m_offsets.push_back(OsrmMappingTypes::SegOffset(nodeId, off)); } } void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp index 12661dab26..8154b6fc31 100644 --- a/routing/osrm2feature_map.hpp +++ b/routing/osrm2feature_map.hpp @@ -9,17 +9,14 @@ #include "../3party/succinct/elias_fano_compressed_list.hpp" - +#include "../defines.hpp" namespace routing { typedef uint32_t OsrmNodeIdT; extern OsrmNodeIdT const INVALID_NODE_ID; -class OsrmFtSegMapping -{ -public: - +namespace OsrmMappingTypes { #pragma pack (push, 1) struct FtSeg { @@ -76,7 +73,6 @@ public: friend string DebugPrint(SegOffset const & off); }; -#pragma pack (pop) struct FtSegLess { @@ -85,8 +81,119 @@ public: return a->Store() < b->Store(); } }; +#pragma pack (pop) +} - typedef set FtSegSetT; +class OsrmFtSegBackwardIndex +{ + typedef pair IndexRecordTypeT; + vector m_index; + + void Save(FilesMappingContainer const & parentCont) + { + const indexName = parentCont.GetName()+FTSEG_MAPPING_BACKWARD_INDEX; + Platform & pl = GetPlatform(); + string const fPath = pl.WritablePathForFile(indexName); + if (!pl.IsFileExistsByFullPath(fPath)) + { + return; + } + + FilesContainerW container(fName); + FileWriter w = container.GetWriter(VERSION_FILE_TAG); + WriterSrc wrt(w); + + FileReader r2 = parentCont.GetReader(VERSION_FILE_TAG); + ReaderSrc src2(r2); + + ver::WriteTimestamp(wrt, ver::ReadTimestamp(src2)); + + size_t count = static_cast(m_index.size()); + FileWriter w2 = container.GetWriter(FTSEG_MAPPING_BACKWARD_INDEX); + WriterSrc wrt2(w2); + WriteVarUint(wrt2, count); + for(size_t i = 0; i < count; ++i) + { + WriteVarUint(src, m_index[i].first); + WriteVarUint(src, m_index[i].second); + } + } + + bool Load(FilesMappingContainer const & parentCont) + { + const indexName = parentCont.GetName()+FTSEG_MAPPING_BACKWARD_INDEX; + Platform & pl = GetPlatform(); + string const fPath = pl.WritablePathForFile(indexName); + if (!pl.IsFileExistsByFullPath(fPath)) + { + return false; + } + // Open new container and check that file has equal timestamp. + FilesMappingContainer container; + container.Open(fPath); + { + FileReader r1 = container.GetReader(VERSION_FILE_TAG); + ReaderSrc src1(r1); + ModelReaderPtr r2 = parentCont.GetReader(VERSION_FILE_TAG); + ReaderSrc src2(r2.GetPtr()); + + if (ver::ReadTimestamp(src1) != ver::ReadTimestamp(src2)) + return false; + } + + FileReader r = container.GetReader(FTSEG_MAPPING_BACKWARD_INDEX); + ReaderSource src(r); + uint32_t const count = ReadVarUint(src); + for(size_t i = 0; i < count; ++i) + { + uint32_t fid = ReadVarUint(src); + uint32_t index = ReadVarUint(src); + m_index[i] = make_pair(fid, index); + } + return true; + } + +public: + void Construct(succinct::elias_fano_compressed_list const & segments, FilesMappingContainer const & parentCont) + { + if (Load(parentCont)) + return; + size_t const count = segments.size(); + m_index.resize(count); + for (size_t i = 0; i < count; ++i) + { + OsrmMappingTypes::FtSeg s(segments[i]); + m_index[i] = make_pair(s.m_fid, i); + } + sort(m_index.begin(), m_index.end(), [](IndexRecordTypeT const & a, IndexRecordTypeT const & b) + { + return a.first < b.first; + }); + Save(parentCont); + } + + void GetIndexesByFid(uint32_t const fid, vector & results) const + { + size_t const start = distance(m_index.begin(), lower_bound(m_index.begin(), m_index.end(), make_pair(fid, 0), + [](IndexRecordTypeT const & a, IndexRecordTypeT const & b) + { + return a.first < b.first; + })); + size_t stop = start; + while (stop FtSegSetT; void Clear(); void Load(FilesMappingContainer & cont); @@ -100,7 +207,7 @@ public: pair r = GetSegmentsRange(nodeId); while (r.first != r.second) { - FtSeg s(m_segments[r.first]); + OsrmMappingTypes::FtSeg s(m_segments[r.first]); if (s.IsValid()) toDo(s); ++r.first; @@ -110,7 +217,7 @@ public: typedef unordered_map > OsrmNodesT; void GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, volatile bool const & requestCancel) const; - void GetSegmentByIndex(size_t idx, FtSeg & seg) const; + void GetSegmentByIndex(size_t idx, OsrmMappingTypes::FtSeg & seg) const; /// @name For debug purpose only. //@{ @@ -129,21 +236,21 @@ public: //@} protected: - typedef vector SegOffsetsT; + typedef vector SegOffsetsT; SegOffsetsT m_offsets; private: succinct::elias_fano_compressed_list m_segments; FilesMappingContainer::Handle m_handle; + OsrmFtSegBackwardIndex m_backwardIndex; }; - class OsrmFtSegMappingBuilder : public OsrmFtSegMapping { public: OsrmFtSegMappingBuilder(); - typedef vector FtSegVectorT; + typedef vector FtSegVectorT; void Append(OsrmNodeIdT nodeId, FtSegVectorT const & data); void Save(FilesContainerW & cont) const; diff --git a/routing/osrm_router.cpp b/routing/osrm_router.cpp index be551328ab..6d012fbaad 100644 --- a/routing/osrm_router.cpp +++ b/routing/osrm_router.cpp @@ -88,7 +88,7 @@ class Point2PhantomNode : private noncopyable uint32_t m_fid; m2::PointD m_point; - Candidate() : m_dist(numeric_limits::max()), m_fid(OsrmFtSegMapping::FtSeg::INVALID_FID) {} + Candidate() : m_dist(numeric_limits::max()), m_fid(OsrmMappingTypes::FtSeg::INVALID_FID) {} }; m2::PointD m_point; @@ -148,11 +148,11 @@ public: } } - if (res.m_fid != OsrmFtSegMapping::FtSeg::INVALID_FID) + if (res.m_fid != OsrmMappingTypes::FtSeg::INVALID_FID) m_candidates.push_back(res); } - double CalculateDistance(OsrmFtSegMapping::FtSeg const & s) const + double CalculateDistance(OsrmMappingTypes::FtSeg const & s) const { ASSERT_NOT_EQUAL(s.m_pointStart, s.m_pointEnd, ()); @@ -173,14 +173,14 @@ public: return dist; } - void CalculateOffset(OsrmFtSegMapping::FtSeg const & seg, m2::PointD const & segPt, NodeID & nodeId, int & offset, bool forward) const + void CalculateOffset(OsrmMappingTypes::FtSeg const & seg, m2::PointD const & segPt, NodeID & nodeId, int & offset, bool forward) const { if (nodeId == INVALID_NODE_ID) return; double distance = 0; auto const range = m_mapping.GetSegmentsRange(nodeId); - OsrmFtSegMapping::FtSeg s, cSeg; + OsrmMappingTypes::FtSeg s, cSeg; int si = forward ? range.second - 1 : range.first; int ei = forward ? range.first - 1 : range.second; @@ -263,7 +263,7 @@ public: if (m_mwmId == numeric_limits::max()) return; - vector segments; + vector segments; segments.resize(maxCount); @@ -276,7 +276,7 @@ public: size_t const n = min(m_candidates.size(), maxCount); for (size_t j = 0; j < n; ++j) { - OsrmFtSegMapping::FtSeg & seg = segments[j]; + OsrmMappingTypes::FtSeg & seg = segments[j]; Candidate const & c = m_candidates[j]; seg.m_fid = c.m_fid; @@ -308,7 +308,7 @@ public: if (!m_direction.IsAlmostZero()) { // Filter income nodes by direction mode - OsrmFtSegMapping::FtSeg const & node_seg = segments[idx]; + OsrmMappingTypes::FtSeg const & node_seg = segments[idx]; FeatureType feature; Index::FeaturesLoaderGuard loader(*m_pIndex, m_mwmId); loader.GetFeature(node_seg.m_fid, feature); @@ -568,7 +568,7 @@ void OsrmRouter::GenerateRoutingTaskFromNodeId(size_t const nodeId, FeatureGraph taskNode.m_node.forward_offset = 0; taskNode.m_node.reverse_offset = 0; taskNode.m_node.name_id = 1; - taskNode.m_seg.m_fid = OsrmFtSegMapping::FtSeg::INVALID_FID; + taskNode.m_seg.m_fid = OsrmMappingTypes::FtSeg::INVALID_FID; } size_t OsrmRouter::FindNextMwmNode(OutgoingCrossNode const & startNode, RoutingMappingPtrT const & targetMapping) @@ -596,6 +596,7 @@ OsrmRouter::ResultCode OsrmRouter::MakeRouteFromCrossesPath(CheckedPathT const & RawRoutingResultT routingResult; FeatureGraphNodeVecT startTask(1), targetTask(1); startTask[0] = cross.startNode; + if (!cross.startNode.m_seg.IsValid()) { startTask.push_back(cross.startNode); @@ -604,6 +605,7 @@ OsrmRouter::ResultCode OsrmRouter::MakeRouteFromCrossesPath(CheckedPathT const & } targetTask[0] = cross.targetNode; + if (!cross.targetNode.m_seg.IsValid()) { targetTask.push_back(cross.targetNode); @@ -1074,7 +1076,7 @@ OsrmRouter::ResultCode OsrmRouter::CalculateRouteImpl(m2::PointD const & startPt } } -m2::PointD OsrmRouter::GetPointForTurnAngle(OsrmFtSegMapping::FtSeg const & seg, +m2::PointD OsrmRouter::GetPointForTurnAngle(OsrmMappingTypes::FtSeg const & seg, FeatureType const & ft, m2::PointD const & turnPnt, size_t (*GetPndInd)(const size_t, const size_t, const size_t)) const { @@ -1102,7 +1104,7 @@ m2::PointD OsrmRouter::GetPointForTurnAngle(OsrmFtSegMapping::FtSeg const & seg, OsrmRouter::ResultCode OsrmRouter::MakeTurnAnnotation(RawRoutingResultT const & routingResult, RoutingMappingPtrT const & mapping, vector & points, Route::TurnsT & turnsDir,Route::TimesT & times, turns::TurnsGeomT & turnsGeom) { - typedef OsrmFtSegMapping::FtSeg SegT; + typedef OsrmMappingTypes::FtSeg SegT; SegT const & segBegin = routingResult.m_sourceEdge.m_seg; SegT const & segEnd = routingResult.m_targetEdge.m_seg; @@ -1157,7 +1159,7 @@ OsrmRouter::ResultCode OsrmRouter::MakeTurnAnnotation(RawRoutingResultT const & auto FindIntersectingSeg = [&buffer] (SegT const & seg) -> size_t { ASSERT(seg.IsValid(), ()); - auto const it = find_if(buffer.begin(), buffer.end(), [&seg] (OsrmFtSegMapping::FtSeg const & s) + auto const it = find_if(buffer.begin(), buffer.end(), [&seg] (OsrmMappingTypes::FtSeg const & s) { return s.IsIntersect(seg); }); @@ -1327,7 +1329,7 @@ void OsrmRouter::GetPossibleTurns(NodeID node, ASSERT_NOT_EQUAL(trg, SPECIAL_NODEID, ()); auto const range = routingMapping->m_segMapping.GetSegmentsRange(trg); - OsrmFtSegMapping::FtSeg seg; + OsrmMappingTypes::FtSeg seg; routingMapping->m_segMapping.GetSegmentByIndex(range.first, seg); if (!seg.IsValid()) continue; @@ -1503,7 +1505,7 @@ void OsrmRouter::GetTurnDirection(PathData const & node1, ASSERT_GREATER(nSegs1.second, 0, ()); - OsrmFtSegMapping::FtSeg seg1, seg2; + OsrmMappingTypes::FtSeg seg1, seg2; routingMapping->m_segMapping.GetSegmentByIndex(nSegs1.second - 1, seg1); routingMapping->m_segMapping.GetSegmentByIndex(nSegs2.first, seg2); diff --git a/routing/osrm_router.hpp b/routing/osrm_router.hpp index 2e3c7fab6b..df5221fdf2 100644 --- a/routing/osrm_router.hpp +++ b/routing/osrm_router.hpp @@ -35,7 +35,7 @@ typedef OsrmDataFacade DataFacadeT; struct FeatureGraphNode { PhantomNode m_node; - OsrmFtSegMapping::FtSeg m_seg; + OsrmMappingTypes::FtSeg m_seg; m2::PointD m_segPt; }; /// All edges available for start route while routing @@ -208,7 +208,7 @@ private: Route::TurnItem & turn); void CalculateTurnGeometry(vector const & points, Route::TurnsT const & turnsDir, turns::TurnsGeomT & turnsGeom) const; void FixupTurns(vector const & points, Route::TurnsT & turnsDir) const; - m2::PointD GetPointForTurnAngle(OsrmFtSegMapping::FtSeg const & seg, + m2::PointD GetPointForTurnAngle(OsrmMappingTypes::FtSeg const & seg, FeatureType const & ft, m2::PointD const & turnPnt, size_t (*GetPndInd)(const size_t, const size_t, const size_t)) const; turns::TurnDirection InvertDirection(turns::TurnDirection dir) const; diff --git a/routing/routing_tests/osrm_router_test.cpp b/routing/routing_tests/osrm_router_test.cpp index b8bca5f8ff..f625dff9e9 100644 --- a/routing/routing_tests/osrm_router_test.cpp +++ b/routing/routing_tests/osrm_router_test.cpp @@ -63,7 +63,7 @@ void TestMapping(InputDataT const & data, { OsrmNodeIdT const node = mapping.GetNodeId(i); size_t count = 0; - mapping.ForEachFtSeg(node, [&] (OsrmFtSegMapping::FtSeg const & s) + mapping.ForEachFtSeg(node, [&] (OsrmMappingTypes::FtSeg const & s) { TEST_EQUAL(s, data[node][count++], ()); }); @@ -74,7 +74,7 @@ void TestMapping(InputDataT const & data, FileWriter::DeleteFileX(fName); } -typedef OsrmFtSegMapping::FtSeg SegT; +typedef OsrmMappingTypes::FtSeg SegT; bool TestFtSeg(SegT const & s) {