pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[indexer] Removed DeltaEncode/DeltaDecode.

Browse source
This commit is contained in:
Maksim Andrianov 2019-11-11 12:16:25 +03:00 • committed by mpimenov
parent 66d7f10403
commit a35ee7a29c
4 changed files with 6 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
indexer/complex/serdes.hpp
View file

@ -13,8 +13,6 @@
#include <string>
#include <vector>
#include <boost/optional.hpp>
namespace complex
{
class ComplexSerdes
@ -43,7 +41,7 @@ public:
{
ReaderSource<decltype(reader)> src(reader);
auto const version = DeserializeVersion(src);
return version ? Deserialize(src, *version, forest) : false;
return Deserialize(src, version, forest);
}
template <typename Sink>
@ -94,14 +92,8 @@ private:
template <typename Sink>
static void Serialize(Sink & sink, tree_node::types::Ptr<Ids> const & tree)
{
uint32_t const base = tree_node::Min(tree, [](auto const & data) {
auto const it = std::min_element(std::cbegin(data), std::cend(data));
CHECK(it != std::cend(data), ());
return *it;
});
WriteVarUint(sink, base);
tree_node::PreOrderVisit(tree, [&](auto const & node) {
coding_utils::DeltaEncode(sink, node->GetData(), base);
coding_utils::WriteCollectionPrimitive(sink, node->GetData());
auto const size = base::checked_cast<coding_utils::CollectionSizeType>(node->GetChildren().size());
WriteVarUint(sink, size);
});
@ -110,14 +102,10 @@ private:
template <typename Src>
static bool Deserialize(Src & src, tree_node::types::Ptr<Ids> & tree)
{
auto const base = ReadVarUint<uint32_t>(src);
std::function<bool(tree_node::types::Ptr<Ids> &)> deserializeTree;
deserializeTree = [&](auto & tree) {
if (src.Size() == 0)
return true;
Ids ids;
coding_utils::DeltaDecode(src, std::back_inserter(ids), base);
coding_utils::ReadCollectionPrimitive(src, std::back_inserter(ids));
tree = tree_node::MakeTreeNode(std::move(ids));
auto const size = ReadVarUint<coding_utils::CollectionSizeType>(src);
tree_node::types::Ptrs<Ids> children(size);
@ -142,14 +130,9 @@ private:
}
template <typename Src>
static boost::optional<Version> DeserializeVersion(Src & src)
static Version DeserializeVersion(Src & src)
{
if (src.Size() < sizeof(kLatestVersion))
{
LOG(LERROR, ("Unable to deserialize complexes: wrong header version."));
return {};
}
return static_cast<Version>(ReadPrimitiveFromSource<std::underlying_type<Version>::type>(src));
return static_cast<Version>(ReadPrimitiveFromSource<std::underlying_type_t<Version>>(src));
}
};
} // namespace complex

96
indexer/complex/serdes_utils.hpp
View file

@ -8,102 +8,8 @@
namespace coding_utils
{
namespace detail
{
// WriteVarIntegral is abstraction over WriteVarUint and WriteVarInt functions.
// WriteVarIntegral may be used in generic code.
template <
typename T, typename Sink,
typename std::enable_if_t<std::is_integral<T>::value && std::is_unsigned<T>::value, int> = 0>
void WriteVarIntegral(Sink & dst, T value)
{
WriteVarUint(dst, value);
}
template <
typename T, typename Sink,
typename std::enable_if_t<std::is_integral<T>::value && std::is_signed<T>::value, int> = 0>
void WriteVarIntegral(Sink & dst, T value)
{
WriteVarInt(dst, value);
}
// ReadVarIntegral is abstraction over ReadVarUint and ReadVarInt functions.
// ReadVarIntegral may be used in generic code.
template <
typename T, typename Source,
typename std::enable_if_t<std::is_integral<T>::value && std::is_unsigned<T>::value, int> = 0>
T ReadVarIntegral(Source & src)
{
return ReadVarUint<T>(src);
}
template <
typename T, typename Source,
typename std::enable_if_t<std::is_integral<T>::value && std::is_signed<T>::value, int> = 0>
T ReadVarIntegral(Source & src)
{
return ReadVarInt<T>(src);
}
} // namespace detail
// Type of collection size. It used for reading and writing collections.
using CollectionSizeType = uint32_t;
// DeltaEncodeAs encodes data in the form of differences between sequential data.
// It required sorted |container|. |fn| may used used to access structure fields or data modification.
template <typename ValueType, typename Sink, typename Cont, typename Fn>
void DeltaEncodeAs(Sink & sink, Cont const & container, Fn && fn)
{
ASSERT((std::is_sorted(std::cbegin(container), std::cend(container),
[&](auto const & lhs, auto const & rhs) { return fn(lhs) < fn(rhs); })),
());
auto const contSize = base::checked_cast<CollectionSizeType>(container.size());
WriteVarUint(sink, contSize);
if (contSize == 0)
return;
auto first = std::begin(container);
auto const last = std::end(container);
auto acc = fn(*first);
detail::WriteVarIntegral(sink, acc);
while (++first != last)
{
auto const val = fn(*first);
auto const delta = base::checked_cast<uint64_t>(val - acc);
WriteVarUint(sink, delta);
acc = val;
}
}
// DeltaDecodeAs decodes data from the form of differences between sequential data.
// |fn| may used used to initialize an object or data modification.
template <typename ValueType, typename Source, typename OutIt, typename Fn>
void DeltaDecodeAs(Source & src, OutIt it, Fn && fn)
{
auto contSize = ReadVarUint<CollectionSizeType>(src);
if (contSize == 0)
return;
auto sum = detail::ReadVarIntegral<ValueType>(src);
*it++ = fn(sum);
while (--contSize)
{
sum += base::checked_cast<ValueType>(ReadVarUint<uint64_t>(src));
*it++ = fn(sum);
}
}
template <typename Sink, typename Cont, typename ValueType = typename Cont::value_type>
void DeltaEncode(Sink & sink, Cont const & container, ValueType base = {})
{
DeltaEncodeAs<ValueType>(sink, container, [&](ValueType val) { return val - base; });
}
template <typename Source, typename OutIt, typename ValueType = typename OutIt::container_type::value_type>
void DeltaDecode(Source & src, OutIt it, ValueType base = {})
{
DeltaDecodeAs<ValueType>(src, it, [&](ValueType val) { return val + base; });
}
using CollectionSizeType = uint64_t;
// WriteCollectionPrimitive writes collection. It used WriteToSink function.
template <typename Sink, typename Cont>

10
indexer/complex/tree_node.hpp
View file

@ -198,16 +198,6 @@ size_t CountIf(types::Ptr<Data> const & node, Fn && fn)
return count;
}
template <typename Data, typename Fn>
decltype(auto) Min(types::Ptr<Data> const & node, Fn && fn)
{
auto m = std::numeric_limits<decltype(fn(node->GetData()))>::max();
PreOrderVisit(node, [&](auto const & node) {
m = std::min(fn(node->GetData()), m);
});
return m;
}
template <typename Data>
void Print(types::Ptr<Data> const & node, std::ostream & stream,
std::string prefix = "", bool isTail = true)

56
indexer/indexer_tests/complex_serdes_utils_tests.cpp
View file

@ -15,21 +15,6 @@ namespace
{
using ByteVector = std::vector<uint8_t>;
template <typename T, typename ValueType = typename T::value_type>
void DeltaEncodeDecodelTest(T const & cont, ValueType base = {})
{
ByteVector buffer;
MemWriter<decltype(buffer)> writer(buffer);
WriterSink<decltype(writer)> sink(writer);
coding_utils::DeltaEncode(sink, cont, base);
MemReader reader(buffer.data(), buffer.size());
ReaderSource<decltype(reader)> src(reader);
T res;
coding_utils::DeltaDecode(src, std::inserter(res, std::end(res)), base);
TEST_EQUAL(cont, res, ());
}
template <typename T>
void CollectionPrimitiveTest(T const & cont)
{
@ -45,47 +30,6 @@ void CollectionPrimitiveTest(T const & cont)
TEST_EQUAL(cont, res, ());
}
UNIT_TEST(Utils_DeltaEncodeDecode)
{
{
std::vector<uint32_t> cont;
DeltaEncodeDecodelTest(cont);
}
{
std::list<uint32_t> cont{1, 2, 3, 4};
cont.sort();
DeltaEncodeDecodelTest(cont);
}
{
std::list<uint32_t> cont{6, 7, 30, 40};
cont.sort();
DeltaEncodeDecodelTest(cont, 6);
}
{
std::list<int32_t> cont{-1, -2, -3, -4};
cont.sort();
DeltaEncodeDecodelTest(cont);
}
{
std::vector<uint32_t> cont{1, 2, 3, 4, 32, 124};
std::sort(std::begin(cont), std::end(cont));
DeltaEncodeDecodelTest(cont, 1);
}
{
std::vector<int32_t> cont{-1, -2, -3, -4, 23, 67};
std::sort(std::begin(cont), std::end(cont));
DeltaEncodeDecodelTest(cont);
}
{
std::set<uint64_t> cont{1, 2, 3, 4, 999, 100124, 243};
DeltaEncodeDecodelTest(cont);
}
{
std::set<int64_t> cont{-1, -2, -3, -4};
DeltaEncodeDecodelTest(cont);
}
}
UNIT_TEST(Utils_CollectionPrimitive)
{
{