organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Fix stack overflow in o5m reader

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-11-07 13:18:11 +03:00 • committed by Sergey Yershov
parent 7d331ef83e
commit 7b111c1583
4 changed files with 59 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
generator/generator_tests/osm_o5m_source_test.cpp
View file

@ -12,6 +12,29 @@
using namespace std;
UNIT_TEST(OSM_O5M_Source_StringTableReferenceTest)
{
string data(begin(node3_o5m_data), end(node3_o5m_data));
stringstream ss{data};
TEST_NOT_EQUAL(data.find("Name Value"), string::npos, ());
TEST_EQUAL(data.find("Name Value"), data.rfind("Name Value"), ());
osm::O5MSource dataset([&ss](uint8_t * buffer, size_t size)
{
return ss.read(reinterpret_cast<char *>(buffer), size).gcount();
}, 10 /* buffer size */);
for (auto const & em : dataset)
{
for (auto const & tag : em.Tags())
{
EXPECT_STREQ(tag.key, "name");
EXPECT_STREQ(tag.value, "Name Value");
}
}
}
UNIT_TEST(OSM_O5M_Source_Entity_smoke_test)
{
osm::O5MSource::Entity e{};

21
generator/generator_tests/source_data.cpp
View file

@ -43,6 +43,27 @@ unsigned char const node2_o5m_data[] = /* 93 */
static_assert(sizeof(node2_o5m_data) == 93, "Size check failed");
char const node3_xml_data[] = R"#(<?xml version='1.0' encoding='UTF-8'?>
<?xml version='1.0' encoding='UTF-8'?>
<osm version='0.6'>
<node id='273105' visible='true' lat='0' lon='0'>
<tag k='name' v='Name Value' />
</node>
<node id='273106' visible='true' lat='1' lon='1'>
<tag k='name' v='Name Value' />
</node>
</osm>
)#";
unsigned char const node3_o5m_data[47] = {
0xFF, 0xE0, 0x04, 0x6F, 0x35, 0x6D, 0x32, 0xFF,
0x10, 0x17, 0xA2, 0xAB, 0x21, 0x00, 0x00, 0x00,
0x00, 0x6E, 0x61, 0x6D, 0x65, 0x00, 0x4E, 0x61,
0x6D, 0x65, 0x20, 0x56, 0x61, 0x6C, 0x75, 0x65,
0x00, 0x10, 0x0B, 0x02, 0x00, 0x80, 0xDA, 0xC4,
0x09, 0x80, 0xDA, 0xC4, 0x09, 0x01, 0xFE};
char const way_xml_data[] = R"#(<?xml version='1.0' encoding='UTF-8'?>
<osm version='0.6'>
<node id='273125' visible='true' lat='60.70931726156' lon='135.05094269106' />

2
generator/generator_tests/source_data.hpp
View file

@ -4,6 +4,8 @@ extern char const node_xml_data[];
extern unsigned char const node_o5m_data[92];
extern char const node2_xml_data[];
extern unsigned char const node2_o5m_data[93];
extern char const node3_xml_data[];
extern unsigned char const node3_o5m_data[47];
extern char const way_xml_data[];
extern unsigned char const way_o5m_data[175];
extern char const relation_xml_data[];

24
generator/osm_o5m_source.hpp
View file

@ -147,6 +147,9 @@ public:
}
protected:
// When reading an .o5m coded file, we use a reference table which has 15,000 lines,
// 250+2 characters each (for performance reasons: 256 characters).
static constexpr size_t kTableSize = 15000;
struct StringTableRecord
{
@ -175,13 +178,11 @@ protected:
char const * role = nullptr;
};
// When reading an .o5m coded file, we use a reference table which has 15,000 lines,
// 250+2 characters each (for performance reasons: 256 characters).
// Every string pair we encounter is copied into the table, with one exception: strings pairs
// which are longer than 250 characters are interpreted but not copied into the table.
std::array<StringTableRecord, 15000> m_stringTable;
std::array<char, 1024> m_stringBuffer;
size_t m_stringCurrentIndex = 0;
std::vector<StringTableRecord> m_stringTable{std::vector<StringTableRecord>(kTableSize)};
size_t m_stringCurrentIndex{0};
std::array<char, 1024> m_keyValueBuffer;
StreamBuffer m_buffer;
size_t m_remainder;
@ -391,14 +392,14 @@ public:
// lookup table
if (kv)
{
size_t const idx = (m_stringCurrentIndex + m_stringTable.size() - key) % m_stringTable.size();
size_t const idx = (m_stringCurrentIndex + kTableSize - key) % kTableSize;
kv->key = m_stringTable[idx].key;
kv->value = m_stringTable[idx].value;
}
return this;
}
char * pBuf = m_stringBuffer.data();
char * pBuf = m_keyValueBuffer.data();
size_t sizes[2] = {0, 0};
for (size_t i = 0; i < (single ? 1 : 2); i++)
{
@ -414,17 +415,18 @@ public:
if (sizes[0] + (single ? 0 : sizes[1]) <= StringTableRecord::MaxEntrySize)
{
memmove(m_stringTable[m_stringCurrentIndex].key, m_stringBuffer.data(), sizes[0]);
memmove(m_stringTable[m_stringCurrentIndex].value, m_stringBuffer.data() + sizes[0], sizes[1]);
memmove(m_stringTable[m_stringCurrentIndex].key, m_keyValueBuffer.data(), sizes[0]);
memmove(m_stringTable[m_stringCurrentIndex].value, m_keyValueBuffer.data() + sizes[0],
sizes[1]);
if (kv)
*kv = KeyValue(m_stringTable[m_stringCurrentIndex].key, m_stringTable[m_stringCurrentIndex].value);
if (++m_stringCurrentIndex == m_stringTable.size())
if (++m_stringCurrentIndex == kTableSize)
m_stringCurrentIndex = 0;
return this;
}
if (kv)
*kv = KeyValue(m_stringBuffer.data(), m_stringBuffer.data() + sizes[0]);
*kv = KeyValue(m_keyValueBuffer.data(), m_keyValueBuffer.data() + sizes[0]);
return this;
}