kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

[coding] Added GZip support.

Browse source
This commit is contained in:
Yuri Gorshenin 2017-05-02 15:48:39 +03:00
parent baaddc479a
commit 5c8c7b6260
6 changed files with 169 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
coding/coding_tests/zlib_test.cpp
View file

@ -2,49 +2,86 @@
#include "coding/zlib.hpp"
#include "base/macros.hpp"
#include "base/string_utils.hpp"
#include "std/cstdint.hpp"
#include "std/iterator.hpp"
#include "std/sstream.hpp"
#include "std/string.hpp"
#include "std/utility.hpp"
#include "std/vector.hpp"
using namespace coding;
using Deflate = ZLib::Deflate;
using Inflate = ZLib::Inflate;
pair<Deflate::Format, Inflate::Format> const g_combinations[] = {
{Deflate::Format::ZLib, Inflate::Format::ZLib},
{Deflate::Format::ZLib, Inflate::Format::Both},
{Deflate::Format::GZip, Inflate::Format::GZip},
{Deflate::Format::GZip, Inflate::Format::Both}};
namespace
{
void TestInflateDeflate(string const & original)
void TestDeflateInflate(string const & original)
{
string compressed;
TEST(ZLib::Deflate(original, ZLib::Level::BestCompression, back_inserter(compressed)), ());
for (auto const & p : g_combinations)
{
Deflate const deflate(p.first /* format */, Deflate::Level::BestCompression);
Inflate const inflate(p.second /* format */);
string decompressed;
TEST(ZLib::Inflate(compressed, back_inserter(decompressed)), ());
string compressed;
TEST(deflate(original, back_inserter(compressed)), ());
TEST_EQUAL(original, decompressed, ());
string decompressed;
TEST(inflate(compressed, back_inserter(decompressed)), ());
TEST_EQUAL(original, decompressed, ());
}
}
UNIT_TEST(ZLib_Smoke)
{
Deflate const deflate(Deflate::Format::ZLib, Deflate::Level::BestCompression);
Inflate const inflate(Inflate::Format::ZLib);
{
string s;
TEST(!ZLib::Deflate(nullptr, 0, ZLib::Level::BestCompression, back_inserter(s)), ());
TEST(!ZLib::Deflate(nullptr, 4, ZLib::Level::BestCompression, back_inserter(s)), ());
TEST(!ZLib::Inflate(nullptr, 0, back_inserter(s)), ());
TEST(!ZLib::Inflate(nullptr, 4, back_inserter(s)), ());
TEST(!deflate(nullptr, 0, back_inserter(s)), ());
TEST(!deflate(nullptr, 4, back_inserter(s)), ());
TEST(!inflate(nullptr, 0, back_inserter(s)), ());
TEST(!inflate(nullptr, 4, back_inserter(s)), ());
}
TestInflateDeflate("");
TestInflateDeflate("Hello, World!");
TestDeflateInflate("");
TestDeflateInflate("Hello, World!");
}
UNIT_TEST(ZLib_Large)
{
string original;
{
ostringstream os;
for (size_t i = 0; i < 1000; ++i)
os << i;
original = os.str();
}
for (size_t i = 0; i < 1000; ++i)
original += strings::to_string(i);
TestInflateDeflate(original);
TestDeflateInflate(original);
}
UNIT_TEST(GZip_ForeignData)
{
// To get this array of bytes, type following:
//
// echo -n 'Hello World!' | gzip -c | od -t x1
uint8_t const data[] = {0x1f, 0x8b, 0x08, 0x08, 0x6d, 0x55, 0x08, 0x59, 0x00, 0x03, 0x73,
0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x74, 0x78, 0x74, 0x00, 0xf3,
0x48, 0xcd, 0xc9, 0xc9, 0xd7, 0x51, 0x08, 0xcf, 0x2f, 0xca, 0x49,
0x51, 0x04, 0x00, 0xd0, 0xc3, 0x4a, 0xec, 0x0d, 0x00, 0x00, 0x00};
string s;
Inflate const inflate(Inflate::Format::GZip);
TEST(inflate(data, ARRAY_SIZE(data), back_inserter(s)), ());
TEST_EQUAL(s, "Hello, World!", ());
}
} // namespace

37
coding/zlib.cpp
View file

@ -6,14 +6,15 @@ namespace coding
{
namespace
{
int ToInt(ZLib::Level level)
int ToInt(ZLib::Deflate::Level level)
{
using Level = ZLib::Deflate::Level;
switch (level)
{
case ZLib::Level::NoCompression: return Z_NO_COMPRESSION;
case ZLib::Level::BestSpeed: return Z_BEST_SPEED;
case ZLib::Level::BestCompression: return Z_BEST_COMPRESSION;
case ZLib::Level::DefaultCompression: return Z_DEFAULT_COMPRESSION;
case Level::NoCompression: return Z_NO_COMPRESSION;
case Level::BestSpeed: return Z_BEST_SPEED;
case Level::BestCompression: return Z_BEST_COMPRESSION;
case Level::DefaultCompression: return Z_DEFAULT_COMPRESSION;
}
}
} // namespace
@ -50,10 +51,20 @@ bool ZLib::Processor::BufferIsFull() const
}
// ZLib::Deflate -----------------------------------------------------------------------------------
ZLib::DeflateProcessor::DeflateProcessor(void const * data, size_t size, ZLib::Level level) noexcept
ZLib::DeflateProcessor::DeflateProcessor(Deflate::Format format, Deflate::Level level,
void const * data, size_t size) noexcept
: Processor(data, size)
{
int const ret = deflateInit(&m_stream, ToInt(level));
auto bits = MAX_WBITS;
switch (format)
{
case Deflate::Format::ZLib: break;
case Deflate::Format::GZip: bits = bits | 16; break;
}
int const ret =
deflateInit2(&m_stream, ToInt(level) /* level */, Z_DEFLATED /* method */,
bits /* windowBits */, 8 /* memLevel */, Z_DEFAULT_STRATEGY /* strategy */);
m_init = (ret == Z_OK);
}
@ -70,10 +81,18 @@ int ZLib::DeflateProcessor::Process(int flush)
}
// ZLib::Inflate -----------------------------------------------------------------------------------
ZLib::InflateProcessor::InflateProcessor(void const * data, size_t size) noexcept
ZLib::InflateProcessor::InflateProcessor(Inflate::Format format, void const * data,
size_t size) noexcept
: Processor(data, size)
{
int const ret = inflateInit(&m_stream);
auto bits = MAX_WBITS;
switch (format)
{
case Inflate::Format::ZLib: break;
case Inflate::Format::GZip: bits = bits | 16; break;
case Inflate::Format::Both: bits = bits | 32; break;
}
int const ret = inflateInit2(&m_stream, bits);
m_init = (ret == Z_OK);
}

99
coding/zlib.hpp
View file

@ -10,7 +10,7 @@
namespace coding
{
// Following class is a wrapper around ZLib routines.
// Following classes are wrappers around ZLib routines.
//
// *NOTE* All Inflate() and Deflate() methods may return false in case
// of errors. In this case the output sequence may be already
@ -19,43 +19,75 @@ namespace coding
class ZLib
{
public:
enum class Level
class Inflate
{
NoCompression,
BestSpeed,
BestCompression,
DefaultCompression
public:
enum class Format
{
ZLib,
GZip,
Both
};
explicit Inflate(Format format) noexcept : m_format(format) {}
template <typename OutIt>
bool operator()(void const * data, size_t size, OutIt out) const
{
if (data == nullptr)
return false;
InflateProcessor processor(m_format, data, size);
return Process(processor, out);
}
template <typename OutIt>
bool operator()(string const & s, OutIt out) const
{
return (*this)(s.c_str(), s.size(), out);
}
private:
Format const m_format;
};
template <typename OutIt>
static bool Deflate(void const * data, size_t size, Level level, OutIt out)
class Deflate
{
if (data == nullptr)
return false;
DeflateProcessor processor(data, size, level);
return Process(processor, out);
}
public:
enum class Format
{
ZLib,
GZip
};
template <typename OutIt>
static bool Deflate(string const & s, Level level, OutIt out)
{
return Deflate(s.c_str(), s.size(), level, out);
}
enum class Level
{
NoCompression,
BestSpeed,
BestCompression,
DefaultCompression
};
template <typename OutIt>
static bool Inflate(void const * data, size_t size, OutIt out)
{
if (data == nullptr)
return false;
InflateProcessor processor(data, size);
return Process(processor, out);
}
Deflate(Format format, Level level) noexcept : m_format(format), m_level(level) {}
template <typename OutIt>
static bool Inflate(string const & s, OutIt out)
{
return Inflate(s.c_str(), s.size(), out);
}
template <typename OutIt>
bool operator()(void const * data, size_t size, OutIt out) const
{
if (data == nullptr)
return false;
DeflateProcessor processor(m_format, m_level, data, size);
return Process(processor, out);
}
template <typename OutIt>
bool operator()(string const & s, OutIt out) const
{
return (*this)(s.c_str(), s.size(), out);
}
private:
Format const m_format;
Level const m_level;
};
private:
class Processor
@ -90,7 +122,8 @@ private:
class DeflateProcessor final : public Processor
{
public:
DeflateProcessor(void const * data, size_t size, Level level) noexcept;
DeflateProcessor(Deflate::Format format, Deflate::Level level, void const * data,
size_t size) noexcept;
virtual ~DeflateProcessor() noexcept override;
int Process(int flush);
@ -101,7 +134,7 @@ private:
class InflateProcessor final : public Processor
{
public:
InflateProcessor(void const * data, size_t size) noexcept;
InflateProcessor(Inflate::Format format, void const * data, size_t size) noexcept;
virtual ~InflateProcessor() noexcept override;
int Process(int flush);

6
map/local_ads_manager.cpp
View file

@ -138,8 +138,10 @@ std::vector<uint8_t> SerializeLocalAdsToJSON(std::list<local_ads::Event> const &
std::unique_ptr<char, JSONFreeDeleter> buffer(
json_dumps(root.get(), JSON_COMPACT | JSON_ENSURE_ASCII));
std::vector<uint8_t> result;
coding::ZLib::Deflate(buffer.get(), strlen(buffer.get()), coding::ZLib::Level::BestCompression,
std::back_inserter(result));
using Deflate = coding::ZLib::Deflate;
Deflate deflate(Deflate::Format::ZLib, Deflate::Level::BestCompression);
deflate(buffer.get(), strlen(buffer.get()), std::back_inserter(result));
return result;
}
#endif

7
platform/http_client_curl.cpp
View file

@ -154,9 +154,14 @@ std::string Decompress(std::string const & compressed, std::string const & encod
std::string decompressed;
if (encoding == "deflate")
ZLib::Inflate(compressed, back_inserter(decompressed));
{
ZLib::Inflate inflate(ZLib::Inflate::Format::ZLib);
inflate(compressed, back_inserter(decompressed));
}
else
{
ASSERT(false, ("Unsupported Content-Encoding:", encoding));
}
return decompressed;
}

12
traffic/traffic_info.cpp
View file

@ -360,16 +360,22 @@ void TrafficInfo::SerializeTrafficValues(vector<SpeedGroup> const & values,
}
}
coding::ZLib::Deflate(buf.data(), buf.size(), coding::ZLib::Level::BestCompression,
back_inserter(result));
using Deflate = coding::ZLib::Deflate;
Deflate deflate(Deflate::Format::ZLib, Deflate::Level::BestCompression);
deflate(buf.data(), buf.size(), back_inserter(result));
}
// static
void TrafficInfo::DeserializeTrafficValues(vector<uint8_t> const & data,
vector<SpeedGroup> & result)
{
using Inflate = coding::ZLib::Inflate;
vector<uint8_t> decompressedData;
coding::ZLib::Inflate(data.data(), data.size(), back_inserter(decompressedData));
Inflate inflate(Inflate::Format::ZLib);
inflate(data.data(), data.size(), back_inserter(decompressedData));
MemReaderWithExceptions memReader(decompressedData.data(), decompressedData.size());
ReaderSource<decltype(memReader)> src(memReader);