forked from organicmaps/organicmaps-tmp
O5M reader and tests
This commit is contained in:
Sergey Yershov
Alex Zolotarev
parent
20627e9c49
commit
c923df38a4
4 changed files with 952 additions and 38 deletions
342
generator/generator_tests/osm_o5m_source_test.cpp
Normal file
342
generator/generator_tests/osm_o5m_source_test.cpp
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
//
|
||||
// osm_o5m_source_test.cpp
|
||||
// generator_tool
|
||||
//
|
||||
// Created by Sergey Yershov on 14.04.15.
|
||||
//
|
||||
|
||||
#include "testing/testing.hpp"
|
||||
#include "generator/osm_o5m_source.hpp"
|
||||
|
||||
#include "std/set.hpp"
|
||||
|
||||
|
||||
/*
|
||||
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<osm version='0.6' upload='true' generator='JOSM'>
|
||||
<node id='-273105' action='modify' visible='true' lat='62.18269750679' lon='-134.28965517091'>
|
||||
<tag k='name' v='Продуктовый' />
|
||||
<tag k='opening_hours' v='24/7' />
|
||||
<tag k='shop' v='convenience' />
|
||||
</node>
|
||||
</osm>
|
||||
|
||||
*/
|
||||
|
||||
/* begin binary data: node.o5m */
|
||||
uint8_t node_o5m_data[] = /* 92 */
|
||||
{0xFF,0xE0,0x04,0x6F,0x35,0x6D,0x32,0xFF,0x10,0x51,0xA1,0xAB,0x21,0x00,0xCD
|
||||
,0xE6,0xD7,0x80,0x0A,0xBE,0xCE,0x82,0xD1,0x04,0x00,0x6E,0x61,0x6D,0x65,0x00
|
||||
,0xD0,0x9F,0xD1,0x80,0xD0,0xBE,0xD0,0xB4,0xD1,0x83,0xD0,0xBA,0xD1,0x82,0xD0
|
||||
,0xBE,0xD0,0xB2,0xD1,0x8B,0xD0,0xB9,0x00,0x00,0x6F,0x70,0x65,0x6E,0x69,0x6E
|
||||
,0x67,0x5F,0x68,0x6F,0x75,0x72,0x73,0x00,0x32,0x34,0x2F,0x37,0x00,0x00,0x73
|
||||
,0x68,0x6F,0x70,0x00,0x63,0x6F,0x6E,0x76,0x65,0x6E,0x69,0x65,0x6E,0x63,0x65
|
||||
,0x00,0xFE};
|
||||
/* end binary data. size = 92 bytes */
|
||||
|
||||
|
||||
/*
|
||||
|
||||
<osm version="0.6" generator="CGImap 0.3.3 (31790 thorn-03.openstreetmap.org)" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
|
||||
<node id="513709898" visible="true" version="8" changeset="12059128" timestamp="2012-06-29T18:09:47Z" user="Xmypblu" uid="395071" lat="55.0927062" lon="38.7666704">
|
||||
<tag k="amenity" v="cinema"/>
|
||||
<tag k="name" v="КТ Горизонт"/>
|
||||
</node>
|
||||
</osm>
|
||||
|
||||
*/
|
||||
|
||||
/* begin binary data: node2.o5m */
|
||||
uint8_t node2_o5m_data[] = /* 93 */
|
||||
{0xFF,0xE0,0x04,0x6F,0x35,0x6D,0x32,0xFF,0x10,0x52,0x94,0xDD,0xF4,0xE9,0x03
|
||||
,0x08,0xD6,0xBD,0xEF,0xFE,0x09,0xF0,0x87,0xC0,0x0B,0x00,0xBF,0x8E,0x18,0x00
|
||||
,0x58,0x6D,0x79,0x70,0x62,0x6C,0x75,0x00,0xA0,0xCC,0xDA,0xF1,0x02,0xAC,0xEB
|
||||
,0xB3,0x8D,0x04,0x00,0x61,0x6D,0x65,0x6E,0x69,0x74,0x79,0x00,0x63,0x69,0x6E
|
||||
,0x65,0x6D,0x61,0x00,0x00,0x6E,0x61,0x6D,0x65,0x00,0xD0,0x9A,0xD0,0xA2,0x20
|
||||
,0xD0,0x93,0xD0,0xBE,0xD1,0x80,0xD0,0xB8,0xD0,0xB7,0xD0,0xBE,0xD0,0xBD,0xD1
|
||||
,0x82,0x00,0xFE};
|
||||
/* end binary data. size = 93 bytes */
|
||||
|
||||
|
||||
/*
|
||||
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<osm version='0.6' upload='true' generator='JOSM'>
|
||||
<node id='-273141' action='modify' visible='true' lat='60.73662144558' lon='-135.06327391353' />
|
||||
<node id='-273139' action='modify' visible='true' lat='60.73186582163' lon='-135.05498459176' />
|
||||
<node id='-273137' action='modify' visible='true' lat='60.72928677326' lon='-135.05450504422' />
|
||||
<node id='-273135' action='modify' visible='true' lat='60.72600404848' lon='-135.05101119785' />
|
||||
<node id='-273133' action='modify' visible='true' lat='60.7207107856' lon='-135.0490245009' />
|
||||
<node id='-273131' action='modify' visible='true' lat='60.71776226098' lon='-135.04587318849' />
|
||||
<node id='-273129' action='modify' visible='true' lat='60.71528261036' lon='-135.04464006624' />
|
||||
<node id='-273126' action='modify' visible='true' lat='60.71283628103' lon='-135.0465582564' />
|
||||
<node id='-273125' action='modify' visible='true' lat='60.70931726156' lon='-135.05094269106' />
|
||||
<way id='-273127' action='modify' visible='true'>
|
||||
<nd ref='-273125' />
|
||||
<nd ref='-273126' />
|
||||
<nd ref='-273129' />
|
||||
<nd ref='-273131' />
|
||||
<nd ref='-273133' />
|
||||
<nd ref='-273135' />
|
||||
<nd ref='-273137' />
|
||||
<nd ref='-273139' />
|
||||
<nd ref='-273141' />
|
||||
<tag k='name' v='Yukon River' />
|
||||
<tag k='name:ru' v='Юкон' />
|
||||
<tag k='waterway' v='river' />
|
||||
</way>
|
||||
</osm>
|
||||
|
||||
*/
|
||||
|
||||
/* begin binary data: way.o5m */
|
||||
uint8_t way_o5m_data[] = /* 175 */
|
||||
{0xFF,0xE0,0x04,0x6F,0x35,0x6D,0x32,0xFF,0x10,0x0E,0xE9,0xAB,0x21,0x00,0xC5
|
||||
,0x94,0x88,0x88,0x0A,0x8C,0xB1,0x9D,0xC3,0x04,0x10,0x08,0x04,0x00,0x9C,0x8F
|
||||
,0x0A,0x87,0xE7,0x05,0x10,0x07,0x04,0x00,0xF6,0x4A,0xFD,0x92,0x03,0x10,0x08
|
||||
,0x04,0x00,0xF6,0xA1,0x04,0xF5,0x80,0x04,0x10,0x08,0x04,0x00,0xB4,0xB6,0x02
|
||||
,0x89,0xBB,0x06,0x10,0x08,0x04,0x00,0xB4,0xEC,0x03,0xD9,0xCC,0x03,0x10,0x08
|
||||
,0x04,0x00,0xD6,0xC0,0x01,0xB7,0x83,0x03,0x10,0x08,0x06,0x00,0xDB,0xAB,0x02
|
||||
,0x9F,0xFE,0x02,0x10,0x08,0x02,0x00,0x87,0xAD,0x05,0xEB,0xA5,0x04,0xFF,0x11
|
||||
,0x44,0xCD,0xAB,0x21,0x00,0x0B,0xC9,0xAB,0x21,0x01,0x05,0x03,0x03,0x03,0x03
|
||||
,0x03,0x03,0x00,0x6E,0x61,0x6D,0x65,0x00,0x59,0x75,0x6B,0x6F,0x6E,0x20,0x52
|
||||
,0x69,0x76,0x65,0x72,0x00,0x00,0x6E,0x61,0x6D,0x65,0x3A,0x72,0x75,0x00,0xD0
|
||||
,0xAE,0xD0,0xBA,0xD0,0xBE,0xD0,0xBD,0x00,0x00,0x77,0x61,0x74,0x65,0x72,0x77
|
||||
,0x61,0x79,0x00,0x72,0x69,0x76,0x65,0x72,0x00,0xFE};
|
||||
/* end binary data. size = 175 bytes */
|
||||
|
||||
|
||||
/*
|
||||
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<osm version='0.6' upload='true' generator='JOSM'>
|
||||
<node id='-273196' action='modify' visible='true' lat='60.71960512058' lon='-135.0538199763'>
|
||||
<tag k='name' v='Whitehorse' />
|
||||
<tag k='place' v='town' />
|
||||
</node>
|
||||
<node id='-273174' action='modify' visible='true' lat='60.71893500205' lon='-135.02422504237' />
|
||||
<node id='-273172' action='modify' visible='true' lat='60.70097061781' lon='-134.99915155667' />
|
||||
<node id='-273170' action='modify' visible='true' lat='60.68655163161' lon='-135.03409002034' />
|
||||
<node id='-273168' action='modify' visible='true' lat='60.68836274296' lon='-135.1128728306' />
|
||||
<node id='-273166' action='modify' visible='true' lat='60.70090356772' lon='-135.1290404334' />
|
||||
<node id='-273164' action='modify' visible='true' lat='60.72181641257' lon='-135.11246178985' />
|
||||
<node id='-273162' action='modify' visible='true' lat='60.74043883115' lon='-135.08451101891' />
|
||||
<node id='-273161' action='modify' visible='true' lat='60.73997005387' lon='-135.06327391353' />
|
||||
<way id='-273163' action='modify' visible='true'>
|
||||
<nd ref='-273161' />
|
||||
<nd ref='-273162' />
|
||||
<nd ref='-273164' />
|
||||
<nd ref='-273166' />
|
||||
<nd ref='-273168' />
|
||||
<nd ref='-273170' />
|
||||
<nd ref='-273172' />
|
||||
<nd ref='-273174' />
|
||||
<nd ref='-273161' />
|
||||
</way>
|
||||
<relation id='-273177' action='modify' visible='true'>
|
||||
<member type='way' ref='-273163' role='outer' />
|
||||
<member type='node' ref='-273196' role='' />
|
||||
<tag k='name' v='Whitehorse' />
|
||||
<tag k='place' v='town' />
|
||||
<tag k='type' v='multipolygon' />
|
||||
</relation>
|
||||
</osm>
|
||||
|
||||
*/
|
||||
|
||||
/* begin binary data: relation.o5m*/
|
||||
uint8_t relation_o5m_data[] = /* 224 */
|
||||
{0xFF,0xE0,0x04,0x6F,0x35,0x6D,0x32,0xFF,0x10,0x2B,0xD7,0xAC,0x21,0x00,0xAD
|
||||
,0xCF,0xFC,0x87,0x0A,0xA6,0xCE,0x88,0xC3,0x04,0x00,0x6E,0x61,0x6D,0x65,0x00
|
||||
,0x57,0x68,0x69,0x74,0x65,0x68,0x6F,0x72,0x73,0x65,0x00,0x00,0x70,0x6C,0x61
|
||||
,0x63,0x65,0x00,0x74,0x6F,0x77,0x6E,0x00,0x10,0x07,0x2C,0x00,0x9A,0x90,0x24
|
||||
,0xD9,0x68,0x10,0x08,0x04,0x00,0xDE,0xCD,0x1E,0xF7,0xF6,0x15,0x10,0x08,0x04
|
||||
,0x00,0x91,0xD3,0x2A,0xFB,0xCC,0x11,0x10,0x08,0x04,0x00,0xE7,0x95,0x60,0xFE
|
||||
,0x9A,0x02,0x10,0x08,0x04,0x00,0x97,0xDE,0x13,0xC0,0xA7,0x0F,0x10,0x08,0x04
|
||||
,0x00,0xB6,0x9E,0x14,0xD2,0xC3,0x19,0x10,0x08,0x04,0x00,0xA6,0x8F,0x22,0xE0
|
||||
,0xDD,0x16,0x10,0x07,0x02,0x00,0xA6,0xF6,0x19,0x9F,0x49,0xFF,0x11,0x10,0x95
|
||||
,0xAC,0x21,0x00,0x0B,0x91,0xAC,0x21,0x01,0x03,0x03,0x03,0x03,0x03,0x03,0x1A
|
||||
,0xFF,0x12,0x46,0xB1,0xAC,0x21,0x00,0x11,0x95,0xAC,0x21,0x00,0x31,0x6F,0x75
|
||||
,0x74,0x65,0x72,0x00,0xD7,0xAC,0x21,0x00,0x30,0x00,0x00,0x6E,0x61,0x6D,0x65
|
||||
,0x00,0x57,0x68,0x69,0x74,0x65,0x68,0x6F,0x72,0x73,0x65,0x00,0x00,0x70,0x6C
|
||||
,0x61,0x63,0x65,0x00,0x74,0x6F,0x77,0x6E,0x00,0x00,0x74,0x79,0x70,0x65,0x00
|
||||
,0x6D,0x75,0x6C,0x74,0x69,0x70,0x6F,0x6C,0x79,0x67,0x6F,0x6E,0x00,0xFE};
|
||||
/* end binary data. size = 224 bytes */
|
||||
|
||||
|
||||
UNIT_TEST(OSM_O5M_Source_Node_read_test)
|
||||
{
|
||||
std::string data(std::begin(node2_o5m_data), std::end(node2_o5m_data));
|
||||
std::stringstream ss(data);
|
||||
|
||||
osm::O5MSourceReader dataset([&ss](uint8_t * buffer, size_t size)
|
||||
{
|
||||
return ss.read((char *)buffer, size).gcount();
|
||||
});
|
||||
|
||||
osm::O5MSourceReader::iterator it = dataset.begin();
|
||||
osm::O5MSourceReader::Entity const & em = *it;
|
||||
|
||||
CHECK_EQUAL(em.id, 513709898, ());
|
||||
CHECK_EQUAL(em.user, string("Xmypblu"), ());
|
||||
CHECK_EQUAL(em.uid, 395071, ());
|
||||
CHECK_EQUAL(em.version, 8, ());
|
||||
CHECK_EQUAL(em.changeset, 12059128, ());
|
||||
CHECK_EQUAL(em.lon, 387666704, ());
|
||||
CHECK_EQUAL(em.lat, 550927062, ());
|
||||
|
||||
auto const & tags = em.tags();
|
||||
auto tag_iterator = tags.begin();
|
||||
auto const & tag = *tag_iterator;
|
||||
CHECK_EQUAL(tag.key, string("amenity"), ());
|
||||
CHECK_EQUAL(tag.value, string("cinema"), ());
|
||||
tag_iterator++;
|
||||
CHECK_EQUAL(tag.key, string("name"), ());
|
||||
CHECK_EQUAL(tag.value, string("КТ Горизонт"), ());
|
||||
tag_iterator++;
|
||||
CHECK(!(tag_iterator != tags.end()), ());
|
||||
}
|
||||
|
||||
UNIT_TEST(OSM_O5M_Source_Way_read_test)
|
||||
{
|
||||
std::string data(std::begin(way_o5m_data), std::end(way_o5m_data));
|
||||
std::stringstream ss(data);
|
||||
|
||||
osm::O5MSourceReader dataset([&ss](uint8_t * buffer, size_t size)
|
||||
{
|
||||
return ss.read((char *)buffer, size).gcount();
|
||||
});
|
||||
|
||||
set<int64_t> nodes;
|
||||
|
||||
vector<pair<string, string>> valid_tags = {
|
||||
{"name", "Yukon River"},
|
||||
{"name:ru", "Юкон"},
|
||||
{"waterway", "river"}
|
||||
};
|
||||
|
||||
for (auto const & em : dataset)
|
||||
{
|
||||
switch (em.type) {
|
||||
case osm::O5MSourceReader::O5M_CMD_NODE:
|
||||
{
|
||||
nodes.insert(em.id);
|
||||
for (auto const & tag : em.tags())
|
||||
{
|
||||
CHECK(false, ("Unexpected tag:", tag.key , tag.value));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case osm::O5MSourceReader::O5M_CMD_WAY:
|
||||
{
|
||||
size_t nd_count = 0;
|
||||
size_t tag_count = 0;
|
||||
for (auto const &nd : em.nodes())
|
||||
{
|
||||
nd_count++;
|
||||
CHECK(nodes.count(nd), ());
|
||||
}
|
||||
CHECK_EQUAL(nodes.size(), nd_count, ());
|
||||
for (auto const & tag : em.tags())
|
||||
{
|
||||
CHECK_EQUAL(tag.key, valid_tags[tag_count].first, ());
|
||||
CHECK_EQUAL(tag.value, valid_tags[tag_count].second, ());
|
||||
tag_count++;
|
||||
}
|
||||
CHECK_EQUAL(valid_tags.size(), tag_count, ());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UNIT_TEST(OSM_O5M_Source_Relation_read_test)
|
||||
{
|
||||
std::string data(std::begin(relation_o5m_data), std::end(relation_o5m_data));
|
||||
std::stringstream ss(data);
|
||||
|
||||
osm::O5MSourceReader dataset([&ss](uint8_t * buffer, size_t size)
|
||||
{
|
||||
return ss.read((char *)buffer, size).gcount();
|
||||
});
|
||||
|
||||
set<int64_t> nodes;
|
||||
set<int64_t> entities;
|
||||
|
||||
vector<pair<string, string>> node_valid_tags = {
|
||||
{"name", "Whitehorse"},
|
||||
{"place", "town"}
|
||||
};
|
||||
|
||||
vector<pair<string, string>> relation_valid_tags = {
|
||||
{"name", "Whitehorse"},
|
||||
{"place", "town"},
|
||||
{"type", "multipolygon"}
|
||||
};
|
||||
|
||||
vector<pair<osm::O5MSourceReader::ECommand, string>> relation_members = {
|
||||
{osm::O5MSourceReader::O5M_CMD_WAY, "outer"},
|
||||
{osm::O5MSourceReader::O5M_CMD_NODE, ""}
|
||||
};
|
||||
|
||||
for (auto const & em : dataset)
|
||||
{
|
||||
entities.insert(em.id);
|
||||
|
||||
switch (em.type) {
|
||||
case osm::O5MSourceReader::O5M_CMD_NODE:
|
||||
{
|
||||
nodes.insert(em.id);
|
||||
size_t tag_count = 0;
|
||||
for (auto const & tag : em.tags())
|
||||
{
|
||||
CHECK_EQUAL(tag.key, node_valid_tags[tag_count].first, ());
|
||||
CHECK_EQUAL(tag.value, node_valid_tags[tag_count].second, ());
|
||||
tag_count++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case osm::O5MSourceReader::O5M_CMD_WAY:
|
||||
{
|
||||
size_t nd_count = 0;
|
||||
for (auto const &nd : em.nodes())
|
||||
{
|
||||
nd_count++;
|
||||
CHECK(nodes.count(nd), ());
|
||||
}
|
||||
CHECK_EQUAL(nodes.size(), nd_count, ());
|
||||
for (auto const & tag : em.tags())
|
||||
{
|
||||
CHECK(false, ("Unexpected tag:", tag.key , tag.value));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case osm::O5MSourceReader::O5M_CMD_REL:
|
||||
{
|
||||
size_t member_count = 0;
|
||||
size_t tag_count = 0;
|
||||
for (auto const &member : em.members())
|
||||
{
|
||||
CHECK(entities.count(member.ref), ());
|
||||
CHECK_EQUAL(relation_members[member_count].first, member.type, ("Current member:", member_count));
|
||||
CHECK_EQUAL(relation_members[member_count].second, member.role, ("Current member:", member_count));
|
||||
member_count++;
|
||||
}
|
||||
CHECK_EQUAL(member_count, 2, ());
|
||||
for (auto const & tag : em.tags())
|
||||
{
|
||||
CHECK_EQUAL(tag.key, relation_valid_tags[tag_count].first, ());
|
||||
CHECK_EQUAL(tag.value, relation_valid_tags[tag_count].second, ());
|
||||
tag_count++;
|
||||
}
|
||||
CHECK_EQUAL(relation_valid_tags.size(), tag_count, ());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
567
generator/osm_o5m_source.hpp
Normal file
567
generator/osm_o5m_source.hpp
Normal file
|
|
@ -0,0 +1,567 @@
|
|||
//
|
||||
// osm_o5m_source.hpp
|
||||
// generator
|
||||
//
|
||||
// Created by Sergey Yershov on 14.04.15.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "std/iostream.hpp"
|
||||
#include "std/sstream.hpp"
|
||||
#include "std/vector.hpp"
|
||||
#include "std/iomanip.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <typename ReaderT>
|
||||
class StreamBuffer
|
||||
{
|
||||
using BufferT = std::vector<uint8_t>;
|
||||
|
||||
ReaderT const & m_reader;
|
||||
BufferT m_buffer;
|
||||
size_t const m_buffer_base_size;
|
||||
size_t m_byte_counter;
|
||||
|
||||
BufferT::value_type const * m_position;
|
||||
BufferT::value_type const * m_start;
|
||||
BufferT::value_type const * m_end;
|
||||
|
||||
public:
|
||||
StreamBuffer(ReaderT const &reader, size_t buffer_size)
|
||||
: m_reader(reader)
|
||||
, m_buffer(buffer_size)
|
||||
, m_buffer_base_size(buffer_size)
|
||||
, m_byte_counter(0)
|
||||
{
|
||||
m_position = m_start = &m_buffer.front();
|
||||
m_end = (&m_buffer.back())+1;
|
||||
read_buffer();
|
||||
}
|
||||
|
||||
size_t byte_counter()
|
||||
{
|
||||
size_t b = m_byte_counter;
|
||||
m_byte_counter = 0;
|
||||
return b;
|
||||
}
|
||||
|
||||
inline BufferT::value_type get()
|
||||
{
|
||||
if(m_position == m_end)
|
||||
read_buffer();
|
||||
++m_byte_counter;
|
||||
return *(m_position++);
|
||||
}
|
||||
|
||||
inline BufferT::value_type peek()
|
||||
{
|
||||
if(m_position == m_end)
|
||||
read_buffer();
|
||||
return *m_position;
|
||||
}
|
||||
|
||||
void skip(size_t size = 1)
|
||||
{
|
||||
if(m_position + size >= m_end)
|
||||
{
|
||||
size -= (m_end - m_position);
|
||||
for (read_buffer(); size > m_buffer.size(); size -= m_buffer.size())
|
||||
read_buffer();
|
||||
}
|
||||
m_position += size;
|
||||
m_byte_counter += size;
|
||||
}
|
||||
|
||||
void read(BufferT::value_type * dest, size_t size) {
|
||||
if(m_position + size >= m_end)
|
||||
{
|
||||
size_t remainder = (m_end - m_position);
|
||||
memcpy(dest, m_position, remainder);
|
||||
size -= remainder;
|
||||
dest += remainder;
|
||||
for (read_buffer(); size > m_buffer.size(); size -= m_buffer.size())
|
||||
{
|
||||
memcpy(dest, m_buffer.data(), m_buffer.size());
|
||||
dest += m_buffer.size();
|
||||
read_buffer();
|
||||
}
|
||||
}
|
||||
memcpy(dest, m_position, size);
|
||||
m_position += size;
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
void read_buffer()
|
||||
{
|
||||
size_t readed_bytes = 0;
|
||||
|
||||
if (m_buffer.size() != m_buffer_base_size)
|
||||
m_buffer.resize(m_buffer_base_size);
|
||||
|
||||
readed_bytes = m_reader(m_buffer.data(), m_buffer.size());
|
||||
|
||||
if (readed_bytes != m_buffer.size())
|
||||
{
|
||||
m_buffer.resize(readed_bytes);
|
||||
m_start = &m_buffer.front();
|
||||
m_end = (&m_buffer.back())+1;
|
||||
}
|
||||
m_position = m_start;
|
||||
}
|
||||
};
|
||||
} // anonymouse namespace
|
||||
|
||||
namespace osm
|
||||
{
|
||||
|
||||
template <typename ReaderT>
|
||||
class O5MSource
|
||||
{
|
||||
template<typename TValue>
|
||||
using TReadFunc = std::function<O5MSource *(TValue*)>;
|
||||
|
||||
struct StringTableRec
|
||||
{
|
||||
char ksize;
|
||||
char vsize;
|
||||
unsigned short int align;
|
||||
char k[254];
|
||||
char v[254];
|
||||
};
|
||||
|
||||
struct KeyValue
|
||||
{
|
||||
char const * key = nullptr;
|
||||
char const * value = nullptr;
|
||||
|
||||
KeyValue() = default;
|
||||
KeyValue(char const *k, char const *v) : key(k), value(v) {}
|
||||
};
|
||||
|
||||
struct Member
|
||||
{
|
||||
int64_t ref = 0;
|
||||
uint8_t type = 0;
|
||||
char const * role = nullptr;
|
||||
};
|
||||
|
||||
std::vector<StringTableRec> m_string_table;
|
||||
std::vector<char> m_string_buffer;
|
||||
size_t m_string_current_index;
|
||||
StreamBuffer<ReaderT> m_buffer;
|
||||
size_t m_remainder;
|
||||
int64_t m_delta_state[3];
|
||||
int64_t m_id = 0;
|
||||
int32_t m_lon = 0;
|
||||
int32_t m_lat = 0;
|
||||
uint64_t m_timestamp = 0;
|
||||
uint64_t m_changeset = 0;
|
||||
int64_t m_refarea_size = 0;
|
||||
|
||||
public:
|
||||
enum ECommand
|
||||
{
|
||||
O5M_CMD_END=0xfe,
|
||||
O5M_CMD_NODE=0x10,
|
||||
O5M_CMD_WAY=0x11,
|
||||
O5M_CMD_REL=0x12,
|
||||
O5M_CMD_BBOX=0xdb,
|
||||
O5M_CMD_TSTAMP=0xdc,
|
||||
O5M_CMD_HEADER=0xe0,
|
||||
O5M_CMD_SYNC=0xee,
|
||||
O5M_CMD_JUMP=0xef,
|
||||
O5M_CMD_RESET=0xff
|
||||
};
|
||||
|
||||
template <typename TValue, typename TFunc>
|
||||
class SubElements
|
||||
{
|
||||
O5MSource * m_reader;
|
||||
TFunc m_func;
|
||||
public:
|
||||
|
||||
class iterator
|
||||
{
|
||||
O5MSource * m_reader;
|
||||
TValue m_val;
|
||||
TFunc m_func;
|
||||
public:
|
||||
|
||||
|
||||
iterator() : m_reader(nullptr) {}
|
||||
explicit iterator(O5MSource *reader, TFunc const & func)
|
||||
: m_reader(reader)
|
||||
, m_func(func)
|
||||
{
|
||||
next_val();
|
||||
}
|
||||
|
||||
bool operator != (iterator const &iter) {return m_reader != iter.m_reader;}
|
||||
|
||||
iterator & operator++() { next_val(); return *this; }
|
||||
iterator operator++(int) { next_val(); return *this; }
|
||||
|
||||
void next_val() { m_reader = m_reader ? m_func(&m_val) : nullptr; }
|
||||
|
||||
TValue const & operator*() const { return m_val; }
|
||||
};
|
||||
|
||||
SubElements(O5MSource * reader, TFunc const & func) : m_reader(reader), m_func(func) {}
|
||||
|
||||
void skip() { while(m_reader ? m_func(nullptr) : nullptr); }
|
||||
|
||||
iterator const begin() const { return iterator(m_reader, m_func); }
|
||||
iterator const end() const { return iterator(); }
|
||||
};
|
||||
|
||||
struct Entity
|
||||
{
|
||||
uint8_t type = 0;
|
||||
int64_t id = 0;
|
||||
uint64_t version = 0;
|
||||
int32_t lon = 0;
|
||||
int32_t lat = 0;
|
||||
int64_t timestamp = 0;
|
||||
int64_t changeset = 0;
|
||||
uint64_t uid = 0;
|
||||
char const * user = nullptr;
|
||||
|
||||
using RefsT = SubElements<Member, TReadFunc<Member>>;
|
||||
using NodesT = SubElements<int64_t, TReadFunc<int64_t>>;
|
||||
using TagsT = SubElements<KeyValue, TReadFunc<KeyValue>>;
|
||||
|
||||
RefsT members() const
|
||||
{
|
||||
return RefsT((type == O5M_CMD_REL) ? m_reader : nullptr, [this](Member *val)
|
||||
{
|
||||
return (m_reader) ? m_reader->ReadMember(val) : nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
NodesT nodes() const
|
||||
{
|
||||
return NodesT((type == O5M_CMD_WAY) ? m_reader : nullptr, [this](int64_t *val)
|
||||
{
|
||||
return (m_reader) ? m_reader->ReadNd(val) : nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
TagsT tags() const
|
||||
{
|
||||
members().skip();
|
||||
nodes().skip();
|
||||
bool const validType = (type == O5M_CMD_NODE || type == O5M_CMD_WAY || type == O5M_CMD_REL);
|
||||
return TagsT( validType ? m_reader : nullptr, [this](KeyValue *val)
|
||||
{
|
||||
return (m_reader) ? m_reader->ReadStringPair(val) : nullptr;
|
||||
});
|
||||
}
|
||||
|
||||
void SkipRemainder()
|
||||
{
|
||||
if (!(type == O5M_CMD_NODE || type == O5M_CMD_WAY || type == O5M_CMD_REL))
|
||||
return;
|
||||
if (type == O5M_CMD_WAY)
|
||||
while(m_reader ? m_reader->ReadNd(nullptr) : nullptr);
|
||||
if (type == O5M_CMD_REL)
|
||||
while(m_reader ? m_reader->ReadMember(nullptr) : nullptr);
|
||||
|
||||
while(m_reader ? m_reader->ReadStringPair(nullptr) : nullptr);
|
||||
}
|
||||
|
||||
Entity() : m_reader(nullptr) {}
|
||||
Entity(O5MSource * reader) : m_reader(reader) {}
|
||||
protected:
|
||||
O5MSource * m_reader;
|
||||
};
|
||||
|
||||
uint64_t ReadUInt()
|
||||
{
|
||||
uint8_t b;
|
||||
uint8_t i = 0;
|
||||
uint64_t ret = 0LL;
|
||||
|
||||
do
|
||||
{
|
||||
b = m_buffer.get();
|
||||
ret |= (uint64_t)(b & 0x7f) << (i++ * 7);
|
||||
} while ( b & 0x80 );
|
||||
size_t rb = m_buffer.byte_counter();
|
||||
m_remainder -= rb;
|
||||
m_refarea_size -= rb;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int64_t ReadInt()
|
||||
{
|
||||
uint64_t ret = ReadUInt();
|
||||
return (ret & 1) ? (-int64_t(ret >> 1) - 1) : int64_t(ret >> 1);
|
||||
}
|
||||
|
||||
O5MSource * ReadMember(Member * ref)
|
||||
{
|
||||
if (m_refarea_size <= 0)
|
||||
return nullptr;
|
||||
int64_t delta = ReadInt();
|
||||
KeyValue kv;
|
||||
if(!ReadStringPair(&kv, true /*single string*/))
|
||||
return nullptr;
|
||||
m_delta_state[kv.key[0] - '0'] += delta;
|
||||
if (ref)
|
||||
{
|
||||
ref->ref = m_delta_state[kv.key[0] - '0'];
|
||||
ref->type = kv.key[0] - ' ';
|
||||
ref->role = &kv.key[1];
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
O5MSource * ReadNd(int64_t * nd)
|
||||
{
|
||||
if (m_refarea_size <= 0)
|
||||
return nullptr;
|
||||
(nd ? *nd : m_delta_state[0]) = (m_delta_state[0] += ReadInt());
|
||||
return this;
|
||||
}
|
||||
|
||||
O5MSource * ReadStringPair(KeyValue * kv, bool single = false)
|
||||
{
|
||||
if (m_remainder == 0)
|
||||
return nullptr;
|
||||
|
||||
uint64_t key = ReadUInt();
|
||||
if (key)
|
||||
{
|
||||
// lookup table
|
||||
if (kv)
|
||||
*kv = KeyValue(m_string_table[m_string_current_index-key].k, m_string_table[m_string_current_index-key].v);
|
||||
return this;
|
||||
}
|
||||
|
||||
char *pBuf = m_string_buffer.data();
|
||||
size_t sizes[2] = {0,0};
|
||||
for (size_t i=0; i<(single?1:2); i++)
|
||||
{
|
||||
do
|
||||
{
|
||||
*pBuf = m_buffer.get();
|
||||
sizes[i]++;
|
||||
} while ( *(pBuf++) );
|
||||
}
|
||||
size_t rb = m_buffer.byte_counter();
|
||||
m_remainder -= rb;
|
||||
m_refarea_size -= rb;
|
||||
|
||||
if (sizes[0]+sizes[1] < 500)
|
||||
{
|
||||
memcpy(m_string_table[m_string_current_index].k, m_string_buffer.data(), sizes[0]);
|
||||
memcpy(m_string_table[m_string_current_index].v, m_string_buffer.data()+sizes[0], sizes[1]);
|
||||
size_t key = m_string_current_index++;
|
||||
if (kv)
|
||||
*kv = KeyValue(m_string_table[key].k, m_string_table[key].v);
|
||||
return this;
|
||||
}
|
||||
|
||||
if (kv)
|
||||
*kv = KeyValue(m_string_buffer.data(), m_string_buffer.data() + sizes[0]);
|
||||
return this;
|
||||
}
|
||||
|
||||
void ReadIdAndVersion(Entity * e)
|
||||
{
|
||||
e->id = (m_id += ReadInt());
|
||||
|
||||
if ((e->version = ReadUInt()) == 0)
|
||||
{
|
||||
e->timestamp = 0;
|
||||
e->changeset = 0;
|
||||
e->uid = 0;
|
||||
e->user = nullptr;
|
||||
return;
|
||||
}
|
||||
|
||||
e->timestamp = (m_timestamp += ReadInt());
|
||||
if (m_timestamp)
|
||||
{
|
||||
e->changeset = (m_changeset += ReadInt());
|
||||
KeyValue kv;
|
||||
ReadStringPair(&kv);
|
||||
uint8_t i = 0;
|
||||
do e->uid |= (uint64_t)((*kv.key) & 0x7f) << (i++ * 7);
|
||||
while ( *(kv.key++) & 0x80 );
|
||||
e->user = kv.value;
|
||||
}
|
||||
}
|
||||
|
||||
void ReadLonLat(Entity * e)
|
||||
{
|
||||
e->lon = (m_lon += ReadInt());
|
||||
e->lat = (m_lat += ReadInt());
|
||||
}
|
||||
|
||||
|
||||
O5MSource * ReadEntity(Entity * entity)
|
||||
{
|
||||
|
||||
entity->SkipRemainder();
|
||||
|
||||
entity->type = m_buffer.get();
|
||||
|
||||
if (O5M_CMD_END == entity->type)
|
||||
return nullptr;
|
||||
|
||||
if (O5M_CMD_RESET == entity->type)
|
||||
{
|
||||
Reset();
|
||||
return ReadEntity(entity);
|
||||
}
|
||||
|
||||
entity->uid = 0;
|
||||
entity->user = nullptr;
|
||||
m_refarea_size = 0;
|
||||
|
||||
m_remainder = ReadUInt(); /* entity size */
|
||||
|
||||
switch (entity->type) {
|
||||
case O5M_CMD_NODE: {
|
||||
ReadIdAndVersion(entity);
|
||||
ReadLonLat(entity);
|
||||
} break;
|
||||
case O5M_CMD_WAY: {
|
||||
ReadIdAndVersion(entity);
|
||||
m_refarea_size = ReadUInt();
|
||||
} break;
|
||||
case O5M_CMD_REL: {
|
||||
ReadIdAndVersion(entity);
|
||||
m_refarea_size = ReadUInt();
|
||||
} break;
|
||||
case O5M_CMD_BBOX: {
|
||||
} break;
|
||||
case O5M_CMD_TSTAMP: {
|
||||
ReadUInt();
|
||||
} break;
|
||||
case O5M_CMD_HEADER: {
|
||||
} break;
|
||||
case O5M_CMD_SYNC: {
|
||||
} break;
|
||||
case O5M_CMD_JUMP: {
|
||||
} break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
void InitStringTable()
|
||||
{
|
||||
m_string_current_index = 0;
|
||||
m_string_buffer.resize(1024);
|
||||
m_string_table.resize(15000);
|
||||
}
|
||||
|
||||
void Reset()
|
||||
{
|
||||
m_delta_state[0] = 0;
|
||||
m_delta_state[1] = 0;
|
||||
m_delta_state[2] = 0;
|
||||
m_id = 0;
|
||||
m_lon = 0;
|
||||
m_lat = 0;
|
||||
m_timestamp = 0;
|
||||
m_changeset = 0;
|
||||
m_remainder = 0;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
static char const * cmd_to_text(uint8_t cmd)
|
||||
{
|
||||
switch (cmd) {
|
||||
case O5M_CMD_END: return "O5M_CMD_END"; break;
|
||||
case O5M_CMD_NODE: return "O5M_CMD_NODE"; break;
|
||||
case O5M_CMD_WAY: return "O5M_CMD_WAY"; break;
|
||||
case O5M_CMD_REL: return "O5M_CMD_REL"; break;
|
||||
case O5M_CMD_BBOX: return "O5M_CMD_BBOX"; break;
|
||||
case O5M_CMD_TSTAMP: return "O5M_CMD_TSTAMP"; break;
|
||||
case O5M_CMD_HEADER: return "O5M_CMD_HEADER"; break;
|
||||
case O5M_CMD_SYNC: return "O5M_CMD_SYNC"; break;
|
||||
case O5M_CMD_JUMP: return "O5M_CMD_JUMP"; break;
|
||||
case O5M_CMD_RESET: return "O5M_CMD_RESET"; break;
|
||||
default: return "Unknown command";
|
||||
}
|
||||
}
|
||||
|
||||
class iterator
|
||||
{
|
||||
O5MSource * m_reader;
|
||||
Entity m_entity;
|
||||
|
||||
public:
|
||||
iterator() : m_reader(nullptr), m_entity(nullptr) {}
|
||||
iterator(O5MSource *reader) : m_reader(reader), m_entity(reader)
|
||||
{
|
||||
m_reader->Reset();
|
||||
next_val();
|
||||
}
|
||||
|
||||
bool operator != (iterator const &iter) {return m_reader != iter.m_reader;}
|
||||
iterator & operator++() { next_val(); return *this; }
|
||||
iterator operator++(int) { next_val(); return *this; }
|
||||
|
||||
void next_val() { m_reader = m_reader->ReadEntity(&m_entity) ? m_reader : nullptr; }
|
||||
|
||||
Entity const & operator*() const { return m_entity; }
|
||||
};
|
||||
|
||||
iterator const begin() {return iterator(this);}
|
||||
iterator const end() {return iterator();}
|
||||
|
||||
O5MSource(ReaderT const & reader) : m_buffer(reader, 10000 /* buffer size */)
|
||||
{
|
||||
if (m_buffer.get() != O5M_CMD_RESET)
|
||||
{
|
||||
throw std::runtime_error("Incorrect o5m start");
|
||||
}
|
||||
CheckHeader();
|
||||
InitStringTable();
|
||||
}
|
||||
|
||||
bool CheckHeader()
|
||||
{
|
||||
size_t len = 4;
|
||||
if (m_buffer.get() == O5M_CMD_HEADER && ReadUInt() == len)
|
||||
{
|
||||
std::string sign(len,' ');
|
||||
m_buffer.read((uint8_t *)sign.data(), len);
|
||||
if (sign == "o5m2" || sign == "o5c2")
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
friend std::ostream & operator << (std::ostream & s, O5MSource::Entity const & em)
|
||||
{
|
||||
s << cmd_to_text(em.type) << " ID: " << em.id;
|
||||
if (em.version)
|
||||
{
|
||||
std::time_t timestamp = em.timestamp;
|
||||
std::tm stm = *std::gmtime(×tamp);
|
||||
s << " Version: " << em.version << " timestamp: " << std::put_time(&stm, "%FT%TZ");
|
||||
s << " changeset: " << em.changeset << " uid: " << em.uid << " user: " << em.user;
|
||||
}
|
||||
if (em.type == O5M_CMD_NODE)
|
||||
{
|
||||
s << std::endl << " lon: " << em.lon << " lat: " << em.lat;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
};
|
||||
|
||||
using TReadFunc = std::function<size_t(uint8_t *,size_t)>;
|
||||
typedef O5MSource<TReadFunc> O5MSourceReader;
|
||||
|
||||
} // namespace osm
|
||||
|
|
@ -22,48 +22,11 @@
|
|||
#include "coding/parse_xml.hpp"
|
||||
|
||||
#include "3party/o5mreader/o5mreader.h"
|
||||
#include "generator/source_reader.hpp"
|
||||
|
||||
|
||||
#define DECODE_O5M_COORD(coord) (static_cast<double>(coord) / 1E+7)
|
||||
|
||||
namespace
|
||||
{
|
||||
class SourceReader
|
||||
{
|
||||
string const &m_filename;
|
||||
FILE * m_file;
|
||||
|
||||
public:
|
||||
SourceReader(string const & filename) : m_filename(filename)
|
||||
{
|
||||
if (m_filename.empty())
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from stdin..."));
|
||||
m_file = freopen(NULL, "rb", stdin);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from", filename));
|
||||
m_file = fopen(filename.c_str(), "rb");
|
||||
}
|
||||
}
|
||||
|
||||
~SourceReader()
|
||||
{
|
||||
if (!m_filename.empty())
|
||||
fclose(m_file);
|
||||
}
|
||||
|
||||
inline FILE * Handle() { return m_file; }
|
||||
|
||||
uint64_t Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
return fread(buffer, sizeof(char), bufferSize, m_file);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
namespace feature
|
||||
{
|
||||
|
||||
|
|
|
|||
42
generator/source_reader.hpp
Normal file
42
generator/source_reader.hpp
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
#pragma once
|
||||
|
||||
#include "std/string.hpp"
|
||||
#include "std/cstdio.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
class SourceReader
|
||||
{
|
||||
string const &m_filename;
|
||||
FILE * m_file;
|
||||
|
||||
public:
|
||||
SourceReader(string const & filename) : m_filename(filename)
|
||||
{
|
||||
if (m_filename.empty())
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from stdin..."));
|
||||
m_file = freopen(NULL, "rb", stdin);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from", filename));
|
||||
m_file = fopen(filename.c_str(), "rb");
|
||||
}
|
||||
}
|
||||
|
||||
~SourceReader()
|
||||
{
|
||||
if (!m_filename.empty())
|
||||
fclose(m_file);
|
||||
}
|
||||
|
||||
inline FILE * Handle() { return m_file; }
|
||||
|
||||
uint64_t Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
return fread(buffer, sizeof(char), bufferSize, m_file);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Loading…
Add table
Reference in a new issue