forked from organicmaps/organicmaps
[routing] Use succinct to compress feature segments vector.
This commit is contained in:
vng
Alex Zolotarev
parent
8d367208c6
commit
ac26a6eac8
4 changed files with 132 additions and 81 deletions
|
|
@ -9,6 +9,8 @@
|
|||
#include "../std/fstream.hpp"
|
||||
#include "../std/sstream.hpp"
|
||||
|
||||
#include "../3party/succinct/mapper.hpp"
|
||||
|
||||
|
||||
namespace routing
|
||||
{
|
||||
|
|
@ -23,6 +25,18 @@ OsrmFtSegMapping::FtSeg::FtSeg(uint32_t fid, uint32_t ps, uint32_t pe)
|
|||
CHECK_EQUAL(m_pointEnd, pe, ());
|
||||
}
|
||||
|
||||
OsrmFtSegMapping::FtSeg::FtSeg(uint64_t x)
|
||||
{
|
||||
m_fid = x & 0xFFFFFFFF;
|
||||
m_pointEnd = (x >> 32) & 0xFFFF;
|
||||
m_pointStart = (x >> 48);
|
||||
}
|
||||
|
||||
uint64_t OsrmFtSegMapping::FtSeg::Store() const
|
||||
{
|
||||
return (uint64_t(m_pointStart) << 48) + (uint64_t(m_pointEnd) << 32) + uint64_t(m_fid);
|
||||
}
|
||||
|
||||
bool OsrmFtSegMapping::FtSeg::Merge(FtSeg const & other)
|
||||
{
|
||||
if (other.m_fid != m_fid)
|
||||
|
|
@ -93,38 +107,29 @@ void OsrmFtSegMapping::Load(FilesMappingContainer & cont)
|
|||
h.Unmap();
|
||||
|
||||
m_handle.Assign(cont.Map(ROUTING_FTSEG_FILE_TAG));
|
||||
}
|
||||
|
||||
size_t OsrmFtSegMapping::GetSegmentsCount() const
|
||||
{
|
||||
return m_handle.GetDataCount<FtSeg>();
|
||||
}
|
||||
|
||||
pair<OsrmFtSegMapping::FtSeg const *, size_t> OsrmFtSegMapping::GetSegVector(OsrmNodeIdT nodeId) const
|
||||
{
|
||||
pair<size_t, size_t> const r = GetSegmentsRange(nodeId);
|
||||
FtSeg const * p = GetSegments() + r.first;
|
||||
return make_pair(p, p->m_fid == -1 ? 0 : r.second);
|
||||
ASSERT(m_handle.IsValid(), ());
|
||||
succinct::mapper::map(m_segments, m_handle.GetData<char>());
|
||||
}
|
||||
|
||||
void OsrmFtSegMapping::DumpSegmentsByFID(uint32_t fID) const
|
||||
{
|
||||
#ifdef DEBUG
|
||||
for (size_t i = 0; i < GetSegmentsCount(); ++i)
|
||||
for (size_t i = 0; i < m_segments.size(); ++i)
|
||||
{
|
||||
FtSeg const * p = GetSegments() + i;
|
||||
if (p->m_fid == fID)
|
||||
LOG(LDEBUG, (*p));
|
||||
FtSeg s(m_segments[i]);
|
||||
if (s.m_fid == fID)
|
||||
LOG(LDEBUG, (s));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void OsrmFtSegMapping::DumpSegmentByNode(uint32_t nodeId) const
|
||||
void OsrmFtSegMapping::DumpSegmentByNode(OsrmNodeIdT nodeId) const
|
||||
{
|
||||
#ifdef DEBUG
|
||||
auto const range = GetSegVector(nodeId);
|
||||
for (size_t i = 0; i < range.second; ++i)
|
||||
LOG(LDEBUG, (range.first[i]));
|
||||
ForEachFtSeg(nodeId, [] (FtSeg const & s)
|
||||
{
|
||||
LOG(LDEBUG, (s));
|
||||
});
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -139,7 +144,7 @@ void OsrmFtSegMapping::GetOsrmNode(FtSeg const & seg, OsrmNodeIdT & forward, Osr
|
|||
size_t const count = GetSegmentsCount();
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
FtSeg const & s = GetSegments()[i];
|
||||
FtSeg s(m_segments[i]);
|
||||
if (s.m_fid != seg.m_fid)
|
||||
continue;
|
||||
|
||||
|
|
@ -178,9 +183,9 @@ pair<size_t, size_t> OsrmFtSegMapping::GetSegmentsRange(OsrmNodeIdT nodeId) cons
|
|||
size_t const start = (index > 0 ? m_offsets[index - 1].m_offset + nodeId : nodeId);
|
||||
|
||||
if (index < m_offsets.size() && m_offsets[index].m_nodeId == nodeId)
|
||||
return make_pair(start, m_offsets[index].m_offset + nodeId - start + 1);
|
||||
return make_pair(start, m_offsets[index].m_offset + nodeId + 1);
|
||||
else
|
||||
return make_pair(start, 1);
|
||||
return make_pair(start, start + 1);
|
||||
}
|
||||
|
||||
OsrmNodeIdT OsrmFtSegMapping::GetNodeId(size_t segInd) const
|
||||
|
|
@ -211,12 +216,17 @@ OsrmFtSegMappingBuilder::OsrmFtSegMappingBuilder()
|
|||
|
||||
void OsrmFtSegMappingBuilder::Append(OsrmNodeIdT osrmNodeId, FtSegVectorT const & data)
|
||||
{
|
||||
if (data.empty())
|
||||
m_segments.push_back(FtSeg(-1, 0, 1));
|
||||
else
|
||||
m_segments.insert(m_segments.end(), data.begin(), data.end());
|
||||
size_t const count = data.size();
|
||||
|
||||
if (data.size() > 1)
|
||||
if (count == 0)
|
||||
m_buffer.push_back(FtSeg(FtSeg::INVALID_FID, 0, 1).Store());
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
m_buffer.push_back(data[i].Store());
|
||||
}
|
||||
|
||||
if (count > 1)
|
||||
{
|
||||
m_lastOffset += (data.size() - 1);
|
||||
|
||||
|
|
@ -229,11 +239,16 @@ void OsrmFtSegMappingBuilder::Append(OsrmNodeIdT osrmNodeId, FtSegVectorT const
|
|||
|
||||
void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const
|
||||
{
|
||||
cont.GetWriter(ROUTING_FTSEG_FILE_TAG).Write(
|
||||
m_segments.data(), sizeof(FtSeg) * m_segments.size());
|
||||
|
||||
cont.GetWriter(ROUTING_NODEIND_TO_FTSEGIND_FILE_TAG).Write(
|
||||
m_offsets.data(), sizeof(SegOffset) * m_offsets.size());
|
||||
|
||||
string const fName = cont.GetFileName() + "." ROUTING_FTSEG_FILE_TAG;
|
||||
|
||||
succinct::elias_fano_compressed_list compressed(m_buffer);
|
||||
succinct::mapper::freeze(compressed, fName.c_str());
|
||||
cont.Write(fName, ROUTING_FTSEG_FILE_TAG);
|
||||
|
||||
FileWriter::DeleteFileX(fName);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
#include "../std/vector.hpp"
|
||||
#include "../std/utility.hpp"
|
||||
|
||||
#include "../3party/succinct/elias_fano_compressed_list.hpp"
|
||||
|
||||
|
||||
namespace routing
|
||||
{
|
||||
|
|
@ -23,13 +25,14 @@ public:
|
|||
uint16_t m_pointStart;
|
||||
uint16_t m_pointEnd;
|
||||
|
||||
FtSeg()
|
||||
: m_fid(-1), m_pointStart(-1), m_pointEnd(-1)
|
||||
{
|
||||
}
|
||||
static constexpr uint32_t INVALID_FID = -1;
|
||||
|
||||
FtSeg() {}
|
||||
FtSeg(uint32_t fid, uint32_t ps, uint32_t pe);
|
||||
|
||||
explicit FtSeg(uint64_t x);
|
||||
uint64_t Store() const;
|
||||
|
||||
bool Merge(FtSeg const & other);
|
||||
|
||||
bool operator == (FtSeg const & other) const
|
||||
|
|
@ -49,10 +52,7 @@ public:
|
|||
OsrmNodeIdT m_nodeId;
|
||||
uint32_t m_offset;
|
||||
|
||||
SegOffset()
|
||||
: m_nodeId(0), m_offset(0)
|
||||
{
|
||||
}
|
||||
SegOffset() : m_nodeId(0), m_offset(0) {}
|
||||
|
||||
SegOffset(uint32_t nodeId, uint32_t offset)
|
||||
: m_nodeId(nodeId), m_offset(offset)
|
||||
|
|
@ -64,22 +64,33 @@ public:
|
|||
void Clear();
|
||||
void Load(FilesMappingContainer & cont);
|
||||
|
||||
pair<FtSeg const *, size_t> GetSegVector(OsrmNodeIdT nodeId) const;
|
||||
template <class ToDo> void ForEachFtSeg(OsrmNodeIdT nodeId, ToDo toDo) const
|
||||
{
|
||||
pair<size_t, size_t> r = GetSegmentsRange(nodeId);
|
||||
while (r.first != r.second)
|
||||
{
|
||||
FtSeg s(m_segments[r.first]);
|
||||
if (s.m_fid != FtSeg::INVALID_FID)
|
||||
toDo(s);
|
||||
++r.first;
|
||||
}
|
||||
}
|
||||
void GetOsrmNode(FtSeg const & seg, OsrmNodeIdT & forward, OsrmNodeIdT & reverse) const;
|
||||
|
||||
/// @name For debug purpose only.
|
||||
//@{
|
||||
void DumpSegmentsByFID(uint32_t fID) const;
|
||||
void DumpSegmentByNode(uint32_t nodeId) const;
|
||||
void DumpSegmentByNode(OsrmNodeIdT nodeId) const;
|
||||
//@}
|
||||
|
||||
/// @name For unit test purpose only.
|
||||
//@{
|
||||
/// @return STL-like range [s, e) of segments indexies for passed node.
|
||||
pair<size_t, size_t> GetSegmentsRange(uint32_t nodeId) const;
|
||||
/// @return Node id for segment's index.
|
||||
OsrmNodeIdT GetNodeId(size_t segInd) const;
|
||||
|
||||
FtSeg const * GetSegments() const { return m_handle.GetData<FtSeg>(); }
|
||||
size_t GetSegmentsCount() const;
|
||||
size_t GetSegmentsCount() const { return m_segments.size(); }
|
||||
//@}
|
||||
|
||||
protected:
|
||||
|
|
@ -87,8 +98,8 @@ protected:
|
|||
SegOffsetsT m_offsets;
|
||||
|
||||
private:
|
||||
succinct::elias_fano_compressed_list m_segments;
|
||||
FilesMappingContainer::Handle m_handle;
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -103,8 +114,7 @@ public:
|
|||
void Save(FilesContainerW & cont) const;
|
||||
|
||||
private:
|
||||
FtSegVectorT m_segments;
|
||||
|
||||
vector<uint64_t> m_buffer;
|
||||
uint64_t m_lastOffset;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -242,26 +242,29 @@ void OsrmRouter::CalculateRoute(m2::PointD const & startingPt, ReadyCallback con
|
|||
for (auto i : osrm::irange<std::size_t>(0, rawRoute.unpacked_path_segments.size()))
|
||||
{
|
||||
// Get all the coordinates for the computed route
|
||||
size_t n = rawRoute.unpacked_path_segments[i].size();
|
||||
size_t const n = rawRoute.unpacked_path_segments[i].size();
|
||||
for (size_t j = 0; j < n; ++j)
|
||||
{
|
||||
PathData const & path_data = rawRoute.unpacked_path_segments[i][j];
|
||||
pair<OsrmFtSegMapping::FtSeg const *, size_t> const range = m_mapping.GetSegVector(path_data.node);
|
||||
|
||||
auto correctFn = [&range] (OsrmFtSegMapping::FtSeg const & seg, size_t & ind)
|
||||
typedef OsrmFtSegMapping::FtSeg SegT;
|
||||
buffer_vector<SegT, 8> buffer;
|
||||
m_mapping.ForEachFtSeg(path_data.node, MakeBackInsertFunctor(buffer));
|
||||
|
||||
auto correctFn = [&buffer] (SegT const & seg, size_t & ind)
|
||||
{
|
||||
auto it = find_if(range.first, range.first + range.second, [&] (OsrmFtSegMapping::FtSeg const & s)
|
||||
auto it = find_if(buffer.begin(), buffer.end(), [&seg] (OsrmFtSegMapping::FtSeg const & s)
|
||||
{
|
||||
return s.IsIntersect(seg);
|
||||
});
|
||||
|
||||
ASSERT(it != range.first + range.second, ());
|
||||
ind = distance(range.first, it);
|
||||
ASSERT(it != buffer.end(), ());
|
||||
ind = distance(buffer.begin(), it);
|
||||
};
|
||||
|
||||
//m_mapping.DumpSegmentByNode(path_data.node);
|
||||
|
||||
size_t startK = 0, endK = range.second;
|
||||
size_t startK = 0, endK = buffer.size();
|
||||
if (j == 0)
|
||||
correctFn(segBegin, startK);
|
||||
else if (j == n - 1)
|
||||
|
|
@ -272,7 +275,7 @@ void OsrmRouter::CalculateRoute(m2::PointD const & startingPt, ReadyCallback con
|
|||
|
||||
for (size_t k = startK; k < endK; ++k)
|
||||
{
|
||||
auto const & seg = range.first[k];
|
||||
SegT const & seg = buffer[k];
|
||||
|
||||
FeatureType ft;
|
||||
Index::FeaturesLoaderGuard loader(*m_pIndex, mwmIdStart);
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ namespace
|
|||
|
||||
typedef vector<OsrmFtSegMappingBuilder::FtSegVectorT> InputDataT;
|
||||
typedef vector< vector<OsrmNodeIdT> > NodeIdDataT;
|
||||
typedef pair<size_t, size_t> PR;
|
||||
typedef vector<PR> RangeDataT;
|
||||
typedef vector< pair<size_t, size_t> > RangeDataT;
|
||||
|
||||
|
||||
void TestNodeId(OsrmFtSegMapping const & mapping, NodeIdDataT const & test)
|
||||
|
|
@ -28,7 +27,11 @@ void TestNodeId(OsrmFtSegMapping const & mapping, NodeIdDataT const & test)
|
|||
void TestSegmentRange(OsrmFtSegMapping const & mapping, RangeDataT const & test)
|
||||
{
|
||||
for (OsrmNodeIdT nodeId = 0; nodeId < test.size(); ++nodeId)
|
||||
TEST_EQUAL(mapping.GetSegmentsRange(nodeId), test[nodeId], ());
|
||||
{
|
||||
// Input test range is { start, count } but we should pass [start, end).
|
||||
auto const & r = test[nodeId];
|
||||
TEST_EQUAL(mapping.GetSegmentsRange(nodeId), make_pair(r.first, r.first + r.second), ());
|
||||
}
|
||||
}
|
||||
|
||||
void TestMapping(InputDataT const & data,
|
||||
|
|
@ -59,16 +62,36 @@ void TestMapping(InputDataT const & data,
|
|||
for (size_t i = 0; i < mapping.GetSegmentsCount(); ++i)
|
||||
{
|
||||
OsrmNodeIdT const node = mapping.GetNodeId(i);
|
||||
auto const v = mapping.GetSegVector(node);
|
||||
TEST_EQUAL(v.second, data[node].size(), ());
|
||||
for (size_t j = 0; j < v.second; ++j)
|
||||
TEST_EQUAL(v.first[j], data[node][j], ());
|
||||
size_t count = 0;
|
||||
mapping.ForEachFtSeg(node, [&] (OsrmFtSegMapping::FtSeg const & s)
|
||||
{
|
||||
TEST_EQUAL(s, data[node][count++], ());
|
||||
});
|
||||
TEST_EQUAL(count, data[node].size(), ());
|
||||
}
|
||||
}
|
||||
|
||||
FileWriter::DeleteFileX(fName);
|
||||
}
|
||||
|
||||
typedef OsrmFtSegMapping::FtSeg SegT;
|
||||
|
||||
bool TestFtSeg(SegT const & s)
|
||||
{
|
||||
return (SegT(s.Store()) == s);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
UNIT_TEST(FtSeg_Smoke)
|
||||
{
|
||||
SegT arr[] = {
|
||||
{ 5, 1, 2 },
|
||||
{ 666, 0, 17 },
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
|
||||
TEST(TestFtSeg(arr[i]), (arr[i].Store()));
|
||||
}
|
||||
|
||||
UNIT_TEST(OsrmFtSegMappingBuilder_Smoke)
|
||||
|
|
@ -98,13 +121,13 @@ UNIT_TEST(OsrmFtSegMappingBuilder_Smoke)
|
|||
|
||||
RangeDataT ranges =
|
||||
{
|
||||
PR(0, 1),
|
||||
PR(1, 1),
|
||||
PR(2, 2),
|
||||
PR(4, 1),
|
||||
PR(5, 3),
|
||||
PR(8, 4),
|
||||
PR(12, 1)
|
||||
{ 0, 1 },
|
||||
{ 1, 1 },
|
||||
{ 2, 2 },
|
||||
{ 4, 1 },
|
||||
{ 5, 3 },
|
||||
{ 8, 4 },
|
||||
{ 12, 1 },
|
||||
};
|
||||
|
||||
TestMapping(data, nodeIds, ranges);
|
||||
|
|
@ -140,15 +163,15 @@ UNIT_TEST(OsrmFtSegMappingBuilder_Smoke)
|
|||
|
||||
RangeDataT ranges =
|
||||
{
|
||||
PR(0, 1),
|
||||
PR(1, 1),
|
||||
PR(2, 1),
|
||||
PR(3, 1),
|
||||
PR(4, 1),
|
||||
PR(5, 2),
|
||||
PR(7, 1),
|
||||
PR(8, 3),
|
||||
PR(11, 3)
|
||||
{ 0, 1 },
|
||||
{ 1, 1 },
|
||||
{ 2, 1 },
|
||||
{ 3, 1 },
|
||||
{ 4, 1 },
|
||||
{ 5, 2 },
|
||||
{ 7, 1 },
|
||||
{ 8, 3 },
|
||||
{ 11, 3 },
|
||||
};
|
||||
|
||||
TestMapping(data, nodeIds, ranges);
|
||||
|
|
@ -174,10 +197,10 @@ UNIT_TEST(OsrmFtSegMappingBuilder_Smoke)
|
|||
|
||||
RangeDataT ranges =
|
||||
{
|
||||
PR(0, 2),
|
||||
PR(2, 1),
|
||||
PR(3, 1),
|
||||
PR(4, 2),
|
||||
{ 0, 2 },
|
||||
{ 2, 1 },
|
||||
{ 3, 1 },
|
||||
{ 4, 2 },
|
||||
};
|
||||
|
||||
TestMapping(data, nodeIds, ranges);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue