From ba6e184115ad519e31c889e0eb46de9c8f13f2e3 Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Tue, 17 Mar 2015 10:49:49 +0300 Subject: [PATCH] Backward feature2osrm index bugfix --- routing/osrm2feature_map.cpp | 110 ++++++++++++--------- routing/osrm2feature_map.hpp | 28 +++--- routing/routing_tests/osrm_router_test.cpp | 10 +- 3 files changed, 81 insertions(+), 67 deletions(-) diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp index 6f43a049dd..f36a0db36f 100644 --- a/routing/osrm2feature_map.cpp +++ b/routing/osrm2feature_map.cpp @@ -2,9 +2,10 @@ #include "../defines.hpp" -#include "../coding/varint.hpp" -#include "../coding/internal/file_data.hpp" #include "../coding/file_name_utils.hpp" +#include "../coding/internal/file_data.hpp" +#include "../coding/read_write_utils.hpp" +#include "../coding/varint.hpp" #include "../base/assert.hpp" #include "../base/logging.hpp" @@ -21,7 +22,7 @@ namespace routing { -OsrmNodeIdT const INVALID_NODE_ID = -1; +TOsrmNodeId const INVALID_NODE_ID = -1; OsrmMappingTypes::FtSeg::FtSeg(uint32_t fid, uint32_t ps, uint32_t pe) : m_fid(fid), @@ -116,7 +117,7 @@ void OsrmFtSegMapping::Load(FilesMappingContainer & cont) m_offsets.resize(count); for (uint32_t i = 0; i < count; ++i) { - m_offsets[i].m_nodeId = ReadVarUint(src); + m_offsets[i].m_nodeId = ReadVarUint(src); m_offsets[i].m_offset = ReadVarUint(src); } } @@ -155,7 +156,7 @@ void OsrmFtSegMapping::DumpSegmentsByFID(uint32_t fID) const #endif } -void OsrmFtSegMapping::DumpSegmentByNode(OsrmNodeIdT nodeId) const +void OsrmFtSegMapping::DumpSegmentByNode(TOsrmNodeId nodeId) const { #ifdef DEBUG ForEachFtSeg(nodeId, [] (OsrmMappingTypes::FtSeg const & s) @@ -170,7 +171,7 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola { auto addResFn = [&] (uint64_t seg, size_t idx, bool forward) { - OsrmNodeIdT const nodeId = GetNodeId(idx); + TOsrmNodeId const nodeId = GetNodeId(idx); auto it = res.insert({ seg, { forward ? nodeId : INVALID_NODE_ID, forward ? INVALID_NODE_ID : nodeId } }); if (it.second) @@ -194,32 +195,36 @@ void OsrmFtSegMapping::GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, vola for (auto it = segments.begin(); it != segments.end(); ++it) { OsrmMappingTypes::FtSeg const & seg = *(*it); - uint32_t nodeId = m_backwardIndex.GetNodeIdByFid(seg.m_fid); - auto range = GetSegmentsRange(nodeId); - for (int i = range.first; i != range.second; ++i) + TNodesList const & nodeIds = m_backwardIndex.GetNodeIdByFid(seg.m_fid); + + for (uint32_t nodeId : nodeIds) { - OsrmMappingTypes::FtSeg const s(m_segments[i]); - if (s.m_fid != seg.m_fid) - continue; - - if (s.m_pointStart <= s.m_pointEnd) + auto const & range = GetSegmentsRange(nodeId); + for (int i = range.first; i != range.second; ++i) { - if (seg.m_pointStart >= s.m_pointStart && seg.m_pointEnd <= s.m_pointEnd) + OsrmMappingTypes::FtSeg const s(m_segments[i]); + if (s.m_fid != seg.m_fid) + continue; + + if (s.m_pointStart <= s.m_pointEnd) { - if (addResFn(seg.Store(), i, true)) + if (seg.m_pointStart >= s.m_pointStart && seg.m_pointEnd <= s.m_pointEnd) { - break; + if (addResFn(seg.Store(), i, true)) + { + break; + } } } - } - else - { - if (seg.m_pointStart >= s.m_pointEnd && seg.m_pointEnd <= s.m_pointStart) + else { - if (addResFn(seg.Store(), i, false)) + if (seg.m_pointStart >= s.m_pointEnd && seg.m_pointEnd <= s.m_pointStart) { - break; + if (addResFn(seg.Store(), i, false)) + { + break; + } } } } @@ -233,7 +238,7 @@ void OsrmFtSegMapping::GetSegmentByIndex(size_t idx, OsrmMappingTypes::FtSeg & s OsrmMappingTypes::FtSeg(m_segments[idx]).Swap(seg); } -pair OsrmFtSegMapping::GetSegmentsRange(OsrmNodeIdT nodeId) const +pair OsrmFtSegMapping::GetSegmentsRange(TOsrmNodeId nodeId) const { SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), OsrmMappingTypes::SegOffset(nodeId, 0), [] (OsrmMappingTypes::SegOffset const & o, OsrmMappingTypes::SegOffset const & val) @@ -250,7 +255,7 @@ pair OsrmFtSegMapping::GetSegmentsRange(OsrmNodeIdT nodeId) cons return make_pair(start, start + 1); } -OsrmNodeIdT OsrmFtSegMapping::GetNodeId(size_t segInd) const +TOsrmNodeId OsrmFtSegMapping::GetNodeId(size_t segInd) const { SegOffsetsT::const_iterator it = lower_bound(m_offsets.begin(), m_offsets.end(), OsrmMappingTypes::SegOffset(segInd, 0), [] (OsrmMappingTypes::SegOffset const & o, OsrmMappingTypes::SegOffset const & val) @@ -276,7 +281,7 @@ OsrmFtSegMappingBuilder::OsrmFtSegMappingBuilder() { } -void OsrmFtSegMappingBuilder::Append(OsrmNodeIdT nodeId, FtSegVectorT const & data) +void OsrmFtSegMappingBuilder::Append(TOsrmNodeId nodeId, FtSegVectorT const & data) { size_t const count = data.size(); @@ -326,7 +331,12 @@ void OsrmFtSegBackwardIndex::Save(string const & nodesFileName, string const & b { { string const nodesFileNameTmp = nodesFileName + EXTENSION_TMP; - succinct::mapper::freeze(m_nodeIds, nodesFileNameTmp.c_str()); + FileWriter nodesFile(nodesFileNameTmp); + WriteVarUint(nodesFile, static_cast(m_nodeIds.size())); + for (auto const bucket : m_nodeIds) + { + rw::WriteVectorOfPOD(nodesFile, bucket); + } my::RenameFileX(nodesFileNameTmp, nodesFileName); } { @@ -341,11 +351,19 @@ bool OsrmFtSegBackwardIndex::Load(string const & nodesFileName, string const & b uint64_t size; if (!GetPlatform().GetFileSizeByFullPath(nodesFileName, size) || !GetPlatform().GetFileSizeByFullPath(bitsFileName, size)) return false; - m_pMappedNodes.reset(new MmapReader(nodesFileName)); - m_pMappedBits.reset(new MmapReader(bitsFileName)); + m_mappedBits.reset(new MmapReader(bitsFileName)); - succinct::mapper::map(m_nodeIds, reinterpret_cast(m_pMappedNodes->Data())); - succinct::mapper::map(m_rankIndex, reinterpret_cast(m_pMappedBits->Data())); + { + FileReader nodesFile(nodesFileName); + ReaderSource nodesSource(nodesFile); + uint32_t size = ReadVarUint(nodesSource); + m_nodeIds.resize(size); + for (uint32_t i = 0; i < size; ++i) + { + rw::ReadVectorOfPOD(nodesSource, m_nodeIds[i]); + } + } + succinct::mapper::map(m_rankIndex, reinterpret_cast(m_mappedBits->Data())); return true; } @@ -374,7 +392,7 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u return; // Generate temporary index to speedup processing - unordered_map temporaryBackwardIndex; + unordered_multimap temporaryBackwardIndex; for (uint32_t i = 0; i < maxNodeId; ++i) { auto indexes = mapping.GetSegmentsRange(i); @@ -388,34 +406,35 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u // Create final index vector inIndex(m_table->size(), false); - vector nodeIds; + vector nodeIds; for (size_t i = 0; i < m_table->size(); ++i) { uint64_t fid = m_table->GetFeatureOffset(i); - auto it = temporaryBackwardIndex.find(fid); - if (it != temporaryBackwardIndex.end()) + auto it = temporaryBackwardIndex.equal_range(fid); + if (it.first != it.second) { inIndex[i] = true; - nodeIds.push_back(it->second); + TNodesList nodesList(distance(it.first, it.second)); + for (auto & node: nodesList) + node = (it.first++)->second; + nodeIds.emplace_back(nodesList); } } // Pack and save index - succinct::elias_fano_compressed_list(nodeIds).swap(m_nodeIds); + nodeIds.swap(m_nodeIds); succinct::rs_bit_vector(inIndex).swap(m_rankIndex); LOG(LINFO, ("Writing section to data file", routingName)); Save(bitsFileName, nodesFileName); } -uint32_t OsrmFtSegBackwardIndex::GetNodeIdByFid(const uint32_t fid) const +TNodesList const & OsrmFtSegBackwardIndex::GetNodeIdByFid(const uint32_t fid) const { - if (!m_table) - return INVALID_NODE_ID; + ASSERT(m_table, ()); size_t const index = m_table->GetFeatureIndexbyOffset(fid); - if (index == m_table->size()) - return INVALID_NODE_ID; + ASSERT_LESS(index, m_table->size(), ("Can't find feature index in offsets table")); size_t node_index = m_rankIndex.rank(index); ASSERT_LESS(node_index, m_nodeIds.size(), ()); return m_nodeIds[node_index]; @@ -423,11 +442,10 @@ uint32_t OsrmFtSegBackwardIndex::GetNodeIdByFid(const uint32_t fid) const void OsrmFtSegBackwardIndex::Clear() { - ClearContainer(m_nodeIds); - ClearContainer(m_rankIndex); + m_nodeIds.clear(); + succinct::rs_bit_vector().swap(m_rankIndex); m_table.reset(); - m_pMappedBits.reset(); - m_pMappedNodes.reset(); + m_mappedBits.reset(); } } diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp index 8f0275195d..08bfc450a8 100644 --- a/routing/osrm2feature_map.hpp +++ b/routing/osrm2feature_map.hpp @@ -22,8 +22,9 @@ namespace routing { -typedef uint32_t OsrmNodeIdT; -extern OsrmNodeIdT const INVALID_NODE_ID; +typedef uint32_t TOsrmNodeId; +typedef vector TNodesList; +extern TOsrmNodeId const INVALID_NODE_ID; namespace OsrmMappingTypes { #pragma pack (push, 1) @@ -70,7 +71,7 @@ namespace OsrmMappingTypes { struct SegOffset { - OsrmNodeIdT m_nodeId; + TOsrmNodeId m_nodeId; uint32_t m_offset; SegOffset() : m_nodeId(0), m_offset(0) {} @@ -97,15 +98,10 @@ class OsrmFtSegMapping; class OsrmFtSegBackwardIndex { succinct::rs_bit_vector m_rankIndex; - succinct::elias_fano_compressed_list m_nodeIds; + vector m_nodeIds; unique_ptr m_table; - unique_ptr m_pMappedNodes, m_pMappedBits; - - template void ClearContainer(T & t) - { - T().swap(t); - } + unique_ptr m_mappedBits; void Save(string const & nodesFileName, string const & bitsFileName); @@ -114,7 +110,7 @@ class OsrmFtSegBackwardIndex public: void Construct(OsrmFtSegMapping const & mapping, uint32_t const maxNodeId, FilesMappingContainer & routingFile); - uint32_t GetNodeIdByFid(uint32_t const fid) const; + TNodesList const & GetNodeIdByFid(uint32_t const fid) const; void Clear(); }; @@ -131,7 +127,7 @@ public: void Unmap(); bool IsMapped() const; - template void ForEachFtSeg(OsrmNodeIdT nodeId, ToDo toDo) const + template void ForEachFtSeg(TOsrmNodeId nodeId, ToDo toDo) const { pair r = GetSegmentsRange(nodeId); while (r.first != r.second) @@ -143,7 +139,7 @@ public: } } - typedef unordered_map > OsrmNodesT; + typedef unordered_map > OsrmNodesT; void GetOsrmNodes(FtSegSetT & segments, OsrmNodesT & res, volatile bool const & requestCancel) const; void GetSegmentByIndex(size_t idx, OsrmMappingTypes::FtSeg & seg) const; @@ -151,7 +147,7 @@ public: /// @name For debug purpose only. //@{ void DumpSegmentsByFID(uint32_t fID) const; - void DumpSegmentByNode(OsrmNodeIdT nodeId) const; + void DumpSegmentByNode(TOsrmNodeId nodeId) const; //@} /// @name For unit test purpose only. @@ -159,7 +155,7 @@ public: /// @return STL-like range [s, e) of segments indexies for passed node. pair GetSegmentsRange(uint32_t nodeId) const; /// @return Node id for segment's index. - OsrmNodeIdT GetNodeId(size_t segInd) const; + TOsrmNodeId GetNodeId(size_t segInd) const; size_t GetSegmentsCount() const { return m_segments.size(); } //@} @@ -181,7 +177,7 @@ public: typedef vector FtSegVectorT; - void Append(OsrmNodeIdT nodeId, FtSegVectorT const & data); + void Append(TOsrmNodeId nodeId, FtSegVectorT const & data); void Save(FilesContainerW & cont) const; private: diff --git a/routing/routing_tests/osrm_router_test.cpp b/routing/routing_tests/osrm_router_test.cpp index 6c63ed3d49..27ee8b7ed6 100644 --- a/routing/routing_tests/osrm_router_test.cpp +++ b/routing/routing_tests/osrm_router_test.cpp @@ -24,13 +24,13 @@ namespace { typedef vector InputDataT; -typedef vector< vector > NodeIdDataT; +typedef vector< vector > NodeIdDataT; typedef vector< pair > RangeDataT; typedef OsrmMappingTypes::FtSeg SegT; void TestNodeId(OsrmFtSegMapping const & mapping, NodeIdDataT const & test) { - for (OsrmNodeIdT nodeId = 0; nodeId < test.size(); ++nodeId) + for (TOsrmNodeId nodeId = 0; nodeId < test.size(); ++nodeId) { for (auto idx : test[nodeId]) TEST_EQUAL(nodeId, mapping.GetNodeId(idx), ()); @@ -39,7 +39,7 @@ void TestNodeId(OsrmFtSegMapping const & mapping, NodeIdDataT const & test) void TestSegmentRange(OsrmFtSegMapping const & mapping, RangeDataT const & test) { - for (OsrmNodeIdT nodeId = 0; nodeId < test.size(); ++nodeId) + for (TOsrmNodeId nodeId = 0; nodeId < test.size(); ++nodeId) { // Input test range is { start, count } but we should pass [start, end). auto const & r = test[nodeId]; @@ -81,7 +81,7 @@ void TestMapping(InputDataT const & data, } OsrmFtSegMappingBuilder builder; - for (OsrmNodeIdT nodeId = 0; nodeId < data.size(); ++nodeId) + for (TOsrmNodeId nodeId = 0; nodeId < data.size(); ++nodeId) builder.Append(nodeId, data[nodeId]); TestNodeId(builder, nodeIds); @@ -102,7 +102,7 @@ void TestMapping(InputDataT const & data, for (size_t i = 0; i < mapping.GetSegmentsCount(); ++i) { - OsrmNodeIdT const node = mapping.GetNodeId(i); + TOsrmNodeId const node = mapping.GetNodeId(i); size_t count = 0; mapping.ForEachFtSeg(node, [&] (OsrmMappingTypes::FtSeg const & s) {