pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

New serializer for traffic data

Browse source
This commit is contained in:
Sergey Yershov 2017-03-06 17:58:42 +03:00
parent 366574678d
commit 6c57ac53cd
7 changed files with 162 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
coding/traffic.cpp
View file

@ -5,7 +5,7 @@
namespace coding
{
// static
uint32_t const TrafficGPSEncoder::kLatestVersion = 0;
uint32_t const TrafficGPSEncoder::kLatestVersion = 1;
uint32_t const TrafficGPSEncoder::kCoordBits = 30;
double const TrafficGPSEncoder::kMinDeltaLat = ms::LatLon::kMinLat - ms::LatLon::kMaxLat;
double const TrafficGPSEncoder::kMaxDeltaLat = ms::LatLon::kMaxLat - ms::LatLon::kMinLat;

140
coding/traffic.hpp
View file

@ -25,11 +25,16 @@ public:
{
DataPoint() = default;
DataPoint(uint64_t timestamp, ms::LatLon latLon) : m_timestamp(timestamp), m_latLon(latLon) {}
DataPoint(uint64_t timestamp, ms::LatLon latLon, uint8_t traffic)
: m_timestamp(timestamp), m_latLon(latLon), m_traffic(traffic)
{
}
// Uint64 should be enough for all our use cases.
// It is expected that |m_timestamp| stores time since epoch in seconds.
uint64_t m_timestamp = 0;
ms::LatLon m_latLon = ms::LatLon::Zero();
// TODO: comment the decision
uint8_t m_traffic = 0;
bool operator==(DataPoint const & p) const
{
@ -43,20 +48,53 @@ public:
// Coordinates are truncated and stored as integers. All integers
// are written as varints.
template <typename Writer, typename Collection>
static size_t SerializeDataPoints(uint32_t version, Writer & writer,
Collection const & points)
static size_t SerializeDataPoints(uint32_t version, Writer & writer, Collection const & points)
{
ASSERT_LESS_OR_EQUAL(version, kLatestVersion, ());
switch (version)
{
case 0: return SerializeDataPointsV0(writer, points);
case 1: return SerializeDataPointsV1(writer, points);
default:
ASSERT(false, ("Unexpected serializer version:", version));
break;
}
return 0;
}
// Deserializes the points from |source| and appends them to |result|.
template <typename Source, typename Collection>
static void DeserializeDataPoints(uint32_t version, Source & src, Collection & result)
{
switch (version)
{
case 0: return DeserializeDataPointsV0(src, result);
case 1: return DeserializeDataPointsV1(src, result);
default:
ASSERT(false, ("Unexpected serializer version:", version));
break;
}
}
private:
static uint32_t DoubleToUint32(double x, double min, double max);
static double Uint32ToDouble(uint32_t x, double min, double max);
template <typename Writer, typename Collection>
static size_t SerializeDataPointsV0(Writer & writer, Collection const & points)
{
auto const startPos = writer.Pos();
if (!points.empty())
{
uint64_t const firstTimestamp = points[0].m_timestamp;
uint32_t const firstLat =
DoubleToUint32(points[0].m_latLon.lat, ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
DoubleToUint32(points[0].m_latLon.lat, ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
uint32_t const firstLon =
DoubleToUint32(points[0].m_latLon.lon, ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
DoubleToUint32(points[0].m_latLon.lon, ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
WriteVarUint(writer, firstTimestamp);
WriteVarUint(writer, firstLat);
WriteVarUint(writer, firstLon);
@ -82,16 +120,55 @@ public:
return static_cast<size_t>(writer.Pos() - startPos);
}
// Deserializes the points from |source| and appends them to |result|.
template <typename Source, typename Collection>
static void DeserializeDataPoints(uint32_t version, Source & src, Collection & result)
template <typename Writer, typename Collection>
static size_t SerializeDataPointsV1(Writer & writer, Collection const & points)
{
ASSERT_LESS_OR_EQUAL(version, kLatestVersion, ());
auto const startPos = writer.Pos();
if (!points.empty())
{
uint64_t const firstTimestamp = points[0].m_timestamp;
uint32_t const firstLat =
DoubleToUint32(points[0].m_latLon.lat, ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
uint32_t const firstLon =
DoubleToUint32(points[0].m_latLon.lon, ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
uint32_t const traffic = points[0].m_traffic;
WriteVarUint(writer, firstTimestamp);
WriteVarUint(writer, firstLat);
WriteVarUint(writer, firstLon);
WriteVarUint(writer, traffic);
}
for (size_t i = 1; i < points.size(); ++i)
{
ASSERT_LESS_OR_EQUAL(points[i - 1].m_timestamp, points[i].m_timestamp, ());
uint64_t const deltaTimestamp = points[i].m_timestamp - points[i - 1].m_timestamp;
uint32_t deltaLat = DoubleToUint32(points[i].m_latLon.lat - points[i - 1].m_latLon.lat,
kMinDeltaLat, kMaxDeltaLat);
uint32_t deltaLon = DoubleToUint32(points[i].m_latLon.lon - points[i - 1].m_latLon.lon,
kMinDeltaLon, kMaxDeltaLon);
uint32_t const traffic = points[i - 1].m_traffic;
WriteVarUint(writer, deltaTimestamp);
WriteVarUint(writer, deltaLat);
WriteVarUint(writer, deltaLon);
WriteVarUint(writer, traffic);
}
ASSERT_LESS_OR_EQUAL(writer.Pos() - startPos, numeric_limits<size_t>::max(),
("Too much data."));
return static_cast<size_t>(writer.Pos() - startPos);
}
template <typename Source, typename Collection>
static void DeserializeDataPointsV0(Source & src, Collection & result)
{
bool first = true;
uint64_t lastTimestamp = 0;
double lastLat = 0.0;
double lastLon = 0.0;
uint8_t traffic = 0;
while (src.Size() > 0)
{
@ -99,10 +176,10 @@ public:
{
lastTimestamp = ReadVarUint<uint64_t>(src);
lastLat =
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
lastLon =
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon)));
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon), traffic));
first = false;
}
else
@ -110,14 +187,43 @@ public:
lastTimestamp += ReadVarUint<uint64_t>(src);
lastLat += Uint32ToDouble(ReadVarUint<uint32_t>(src), kMinDeltaLat, kMaxDeltaLat);
lastLon += Uint32ToDouble(ReadVarUint<uint32_t>(src), kMinDeltaLon, kMaxDeltaLon);
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon)));
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon), traffic));
}
}
}
private:
static uint32_t DoubleToUint32(double x, double min, double max);
template <typename Source, typename Collection>
static void DeserializeDataPointsV1(Source & src, Collection & result)
{
bool first = true;
uint64_t lastTimestamp = 0;
double lastLat = 0.0;
double lastLon = 0.0;
uint8_t traffic = 0;
while (src.Size() > 0)
{
if (first)
{
lastTimestamp = ReadVarUint<uint64_t>(src);
lastLat =
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLat, ms::LatLon::kMaxLat);
lastLon =
Uint32ToDouble(ReadVarUint<uint32_t>(src), ms::LatLon::kMinLon, ms::LatLon::kMaxLon);
traffic = static_cast<uint8_t>(ReadVarUint<uint32_t>(src));
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon), traffic));
first = false;
}
else
{
lastTimestamp += ReadVarUint<uint64_t>(src);
lastLat += Uint32ToDouble(ReadVarUint<uint32_t>(src), kMinDeltaLat, kMaxDeltaLat);
lastLon += Uint32ToDouble(ReadVarUint<uint32_t>(src), kMinDeltaLon, kMaxDeltaLon);
traffic = static_cast<uint8_t>(ReadVarUint<uint32_t>(src));
result.push_back(DataPoint(lastTimestamp, ms::LatLon(lastLat, lastLon), traffic));
}
}
}
static double Uint32ToDouble(uint32_t x, double min, double max);
};
} // namespace coding

40
tracking/protocol.cpp
View file

@ -12,15 +12,23 @@
namespace
{
template <typename Container>
vector<uint8_t> CreateDataPacketImpl(Container const & points)
vector<uint8_t> CreateDataPacketImpl(Container const & points,
tracking::Protocol::PacketType const type)
{
vector<uint8_t> buffer;
MemWriter<decltype(buffer)> writer(buffer);
tracking::Protocol::Encoder::SerializeDataPoints(tracking::Protocol::Encoder::kLatestVersion,
writer, points);
auto packet = tracking::Protocol::CreateHeader(tracking::Protocol::PacketType::CurrentData,
static_cast<uint32_t>(buffer.size()));
uint32_t serializer_version = tracking::Protocol::Encoder::kLatestVersion;
switch (type)
{
case tracking::Protocol::PacketType::DataV0: serializer_version = 0; break;
case tracking::Protocol::PacketType::DataV1: serializer_version = 1; break;
case tracking::Protocol::PacketType::AuthV0: ASSERT(false, ("Not a DATA type.")); break;
}
tracking::Protocol::Encoder::SerializeDataPoints(serializer_version, writer, points);
auto packet = tracking::Protocol::CreateHeader(type, static_cast<uint32_t>(buffer.size()));
packet.insert(packet.end(), begin(buffer), end(buffer));
return packet;
@ -54,15 +62,15 @@ vector<uint8_t> Protocol::CreateAuthPacket(string const & clientId)
}
// static
vector<uint8_t> Protocol::CreateDataPacket(DataElementsCirc const & points)
vector<uint8_t> Protocol::CreateDataPacket(DataElementsCirc const & points, PacketType type)
{
return CreateDataPacketImpl(points);
return CreateDataPacketImpl(points, type);
}
// static
vector<uint8_t> Protocol::CreateDataPacket(DataElementsVec const & points)
vector<uint8_t> Protocol::CreateDataPacket(DataElementsVec const & points, PacketType type)
{
return CreateDataPacketImpl(points);
return CreateDataPacketImpl(points, type);
}
// static
@ -82,9 +90,9 @@ string Protocol::DecodeAuthPacket(Protocol::PacketType type, vector<uint8_t> con
{
switch (type)
{
case Protocol::PacketType::AuthV0:
return string(begin(data), end(data));
case Protocol::PacketType::DataV0: break;
case Protocol::PacketType::AuthV0: return string(begin(data), end(data));
case Protocol::PacketType::DataV0:
case Protocol::PacketType::DataV1: ASSERT(false, ("Not AUTH packet.")); break;
}
return string();
}
@ -98,9 +106,12 @@ Protocol::DataElementsVec Protocol::DecodeDataPacket(PacketType type, vector<uin
switch (type)
{
case Protocol::PacketType::DataV0:
Encoder::DeserializeDataPoints(Encoder::kLatestVersion, src, points);
Encoder::DeserializeDataPoints(0 /* version */, src, points);
break;
case Protocol::PacketType::AuthV0: break;
case Protocol::PacketType::DataV1:
Encoder::DeserializeDataPoints(1 /* version */, src, points);
break;
case Protocol::PacketType::AuthV0: ASSERT(false, ("Not DATA packet.")); break;
}
return points;
}
@ -126,6 +137,7 @@ string DebugPrint(Protocol::PacketType type)
{
case Protocol::PacketType::AuthV0: return "AuthV0";
case Protocol::PacketType::DataV0: return "DataV0";
case Protocol::PacketType::DataV1: return "DataV1";
}
stringstream ss;
ss << "Unknown(" << static_cast<uint32_t>(type) << ")";

9
tracking/protocol.hpp
View file

@ -23,15 +23,18 @@ public:
{
AuthV0 = 0x81,
DataV0 = 0x82,
DataV1 = 0x92,
CurrentAuth = AuthV0,
CurrentData = DataV0
CurrentData = DataV1
};
static vector<uint8_t> CreateHeader(PacketType type, uint32_t payloadSize);
static vector<uint8_t> CreateAuthPacket(string const & clientId);
static vector<uint8_t> CreateDataPacket(DataElementsCirc const & points);
static vector<uint8_t> CreateDataPacket(DataElementsVec const & points);
static vector<uint8_t> CreateDataPacket(DataElementsCirc const & points,
PacketType type = PacketType::CurrentData);
static vector<uint8_t> CreateDataPacket(DataElementsVec const & points,
PacketType type = PacketType::CurrentData);
static std::pair<PacketType, size_t> DecodeHeader(vector<uint8_t> const & data);
static string DecodeAuthPacket(PacketType type, vector<uint8_t> const & data);

1
tracking/pytracking/bindings.cpp
View file

@ -34,6 +34,7 @@ BOOST_PYTHON_MODULE(pytracking)
enum_<Protocol::PacketType>("PacketType")
.value("AuthV0", Protocol::PacketType::AuthV0)
.value("DataV0", Protocol::PacketType::DataV0)
.value("DataV1", Protocol::PacketType::DataV1)
.value("CurrentAuth", Protocol::PacketType::CurrentAuth)
.value("CurrentData", Protocol::PacketType::CurrentData);

6
tracking/reporter.cpp
View file

@ -43,7 +43,7 @@ Reporter::~Reporter()
m_thread.join();
}
void Reporter::AddLocation(location::GpsInfo const & info, traffic::SpeedGroup /* speedGroup */)
void Reporter::AddLocation(location::GpsInfo const & info, traffic::SpeedGroup traffic)
{
lock_guard<mutex> lg(m_mutex);
@ -54,7 +54,9 @@ void Reporter::AddLocation(location::GpsInfo const & info, traffic::SpeedGroup /
return;
m_lastGpsTime = info.m_timestamp;
m_input.push_back(DataPoint(info.m_timestamp, ms::LatLon(info.m_latitude, info.m_longitude)));
m_input.push_back(DataPoint(info.m_timestamp,
ms::LatLon(info.m_latitude, info.m_longitude),
static_cast<std::underlying_type<traffic::SpeedGroup>::type>(traffic)));
}
void Reporter::Run()

2
tracking/reporter.hpp
View file

@ -39,7 +39,7 @@ public:
milliseconds pushDelay);
~Reporter();
void AddLocation(location::GpsInfo const & info, traffic::SpeedGroup speedGroup);
void AddLocation(location::GpsInfo const & info, traffic::SpeedGroup traffic);
void SetAllowSendingPoints(bool allow) { m_allowSendingPoints = allow; }