[ugc] Basic SerDes.

This commit is contained in:
Yuri Gorshenin 2017-06-20 16:11:46 +03:00 committed by Yuri Gorshenin
parent f5bdec0ba4
commit f30a26682d
6 changed files with 532 additions and 64 deletions

View file

@ -272,3 +272,9 @@ TPrimitive ReadPrimitiveFromSource(TSource & source)
source.Read(&primitive, sizeof(primitive));
return SwapIfBigEndian(primitive);
}
template <typename TPrimitive, typename TSource>
void ReadPrimitiveFromSource(TSource & source, TPrimitive & primitive)
{
primitive = ReadPrimitiveFromSource<TPrimitive, TSource>(source);
}

View file

@ -4,7 +4,9 @@ set(
SRC
api.cpp
api.hpp
serdes.hpp
types.hpp
)
add_library(${PROJECT_NAME} ${SRC})
omim_add_test_subdirectory(ugc_tests)

232
ugc/serdes.hpp Normal file
View file

@ -0,0 +1,232 @@
#pragma once
#include "ugc/types.hpp"
#include "coding/multilang_utf8_string.hpp"
#include "coding/point_to_integer.hpp"
#include "coding/reader.hpp"
#include "coding/varint.hpp"
#include "coding/write_to_sink.hpp"
#include <cmath>
#include <cstdint>
namespace ugc
{
enum class Version : uint8_t
{
V0
};
struct HeaderV0
{
};
template <typename Sink>
class Serializer
{
public:
Serializer(Sink & sink, HeaderV0 const & header) : m_sink(sink), m_header(header) {}
void operator()(uint8_t const d) { WriteToSink(m_sink, d); }
void operator()(uint32_t const d) { WriteToSink(m_sink, d); }
void operator()(uint64_t const d) { WriteToSink(m_sink, d); }
void operator()(std::string const & s) { utils::WriteString(m_sink, s); }
void SerRating(float const f)
{
CHECK_GREATER_OR_EQUAL(f, 0.0, ());
auto const d = static_cast<uint32_t>(round(f * 10));
SerVarUint(d);
}
template <typename T>
void SerVarUint(T const & t)
{
WriteVarUint(m_sink, t);
}
template <typename T>
void operator()(vector<T> const & vs)
{
SerVarUint(vs.size());
for (auto const & v : vs)
(*this)(v);
}
void operator()(RatingRecord const & ratingRecord)
{
(*this)(ratingRecord.m_key);
SerRating(ratingRecord.m_value);
}
void operator()(Rating const & rating)
{
(*this)(rating.m_ratings);
SerRating(rating.m_aggValue);
}
void operator()(UID const & uid)
{
(*this)(uid.m_hi);
(*this)(uid.m_lo);
}
void operator()(Author const & author)
{
(*this)(author.m_uid);
(*this)(author.m_name);
}
void operator()(Text const & text)
{
(*this)(text.m_lang);
(*this)(text.m_text);
}
void operator()(Review::Sentiment sentiment)
{
switch (sentiment)
{
case Review::Sentiment::Negative: return (*this)(static_cast<uint8_t>(0));
case Review::Sentiment::Positive: return (*this)(static_cast<uint8_t>(1));
}
}
void operator()(Review const & review)
{
(*this)(review.m_id);
(*this)(review.m_text);
(*this)(review.m_author);
SerRating(review.m_rating);
(*this)(review.m_sentiment);
SerVarUint(review.DaysSinceEpoch());
}
void operator()(Attribute const & attribute)
{
(*this)(attribute.m_key);
(*this)(attribute.m_value);
}
void operator()(UGC const & ugc)
{
(*this)(ugc.m_rating);
(*this)(ugc.m_reviews);
(*this)(ugc.m_attributes);
}
private:
Sink & m_sink;
HeaderV0 const m_header;
};
template <typename Source>
class DeserializerV0
{
public:
DeserializerV0(Source & source, HeaderV0 const & header) : m_source(source), m_header(header) {}
void operator()(uint8_t & d) { ReadPrimitiveFromSource(m_source, d); }
void operator()(uint32_t & d) { ReadPrimitiveFromSource(m_source, d); }
void operator()(uint64_t & d) { ReadPrimitiveFromSource(m_source, d); }
void operator()(std::string & s) { utils::ReadString(m_source, s); }
void DesRating(float & f)
{
uint32_t d = 0;
DesVarUint(d);
f = static_cast<float>(d) / 10;
}
template <typename T>
void DesVarUint(T & t)
{
t = ReadVarUint<T, Source>(m_source);
}
template <typename T>
T DesVarUint()
{
return ReadVarUint<T, Source>(m_source);
}
void operator()(RatingRecord & ratingRecord)
{
(*this)(ratingRecord.m_key);
DesRating(ratingRecord.m_value);
}
template <typename T>
void operator()(vector<T> & vs)
{
auto const size = DesVarUint<size_t>();
vs.resize(size);
for (auto & v : vs)
(*this)(v);
}
void operator()(Rating & rating)
{
(*this)(rating.m_ratings);
DesRating(rating.m_aggValue);
}
void operator()(UID & uid)
{
(*this)(uid.m_hi);
(*this)(uid.m_lo);
}
void operator()(Author & author)
{
(*this)(author.m_uid);
(*this)(author.m_name);
}
void operator()(Text & text)
{
(*this)(text.m_lang);
(*this)(text.m_text);
}
void operator()(Review::Sentiment & sentiment)
{
uint8_t s = 0;
(*this)(s);
switch (s)
{
case 0: sentiment = Review::Sentiment::Negative; break;
case 1: sentiment = Review::Sentiment::Positive; break;
default: CHECK(false, ("Can't parse sentiment from:", static_cast<int>(s))); break;
}
}
void operator()(Review & review)
{
(*this)(review.m_id);
(*this)(review.m_text);
(*this)(review.m_author);
DesRating(review.m_rating);
(*this)(review.m_sentiment);
review.SetDaysSinceEpoch(DesVarUint<uint32_t>());
}
void operator()(Attribute & attribute)
{
(*this)(attribute.m_key);
(*this)(attribute.m_value);
}
void operator()(UGC & ugc)
{
(*this)(ugc.m_rating);
(*this)(ugc.m_reviews);
(*this)(ugc.m_attributes);
}
private:
Source & m_source;
HeaderV0 const m_header;
};
} // namespace ugc

View file

@ -2,9 +2,14 @@
#include "indexer/feature_decl.hpp"
#include "coding/hex.hpp"
#include <chrono>
#include <cstdint>
#include <memory>
#include <sstream>
#include <string>
#include <string>
#include <vector>
@ -14,133 +19,233 @@ using TranslationKey = std::string;
struct RatingRecord
{
RatingRecord(TranslationKey const & key, float const value)
: m_key(key)
, m_value(value)
RatingRecord() = default;
RatingRecord(TranslationKey const & key, float const value) : m_key(key), m_value(value) {}
bool operator==(RatingRecord const & rhs) const
{
return m_key == rhs.m_key && m_value == rhs.m_value;
}
TranslationKey m_key;
float m_value;
friend std::string DebugPrint(RatingRecord const & ratingRecord)
{
std::ostringstream os;
os << "RatingRecord [ " << ratingRecord.m_key << " " << ratingRecord.m_value << " ]";
return os.str();
}
TranslationKey m_key{};
float m_value{};
};
struct Rating
{
Rating() = default;
Rating(std::vector<RatingRecord> const & ratings, float const aggValue)
: m_ratings(ratings)
, m_aggValue(aggValue)
: m_ratings(ratings), m_aggValue(aggValue)
{
}
bool operator==(Rating const & rhs) const
{
return m_ratings == rhs.m_ratings && m_aggValue == rhs.m_aggValue;
}
friend std::string DebugPrint(Rating const & rating)
{
std::ostringstream os;
os << "Rating [ ratings:" << DebugPrint(rating.m_ratings) << ", aggValue:" << rating.m_aggValue
<< " ]";
return os.str();
}
std::vector<RatingRecord> m_ratings;
float m_aggValue;
float m_aggValue{};
};
class UID
struct UID
{
public:
UID(uint64_t const hi, uint64_t const lo)
: m_hi(hi)
, m_lo(lo)
UID() = default;
UID(uint64_t const hi, uint64_t const lo) : m_hi(hi), m_lo(lo) {}
std::string ToString() const { return NumToHex(m_hi) + NumToHex(m_lo); }
bool operator==(UID const & rhs) const { return m_hi == rhs.m_hi && m_lo == rhs.m_lo; }
friend std::string DebugPrint(UID const & uid)
{
std::ostringstream os;
os << "UID [ " << uid.ToString() << " ]";
return os.str();
}
std::string ToString() const;
private:
uint64_t m_hi;
uint64_t m_lo;
uint64_t m_hi{};
uint64_t m_lo{};
};
struct Author
{
Author(UID const & uid, std::string const & name)
: m_uid(uid)
, m_name(name)
Author() = default;
Author(UID const & uid, std::string const & name) : m_uid(uid), m_name(name) {}
bool operator==(Author const & rhs) const { return m_uid == rhs.m_uid && m_name == rhs.m_name; }
friend std::string DebugPrint(Author const & author)
{
std::ostringstream os;
os << "Author [ " << DebugPrint(author.m_uid) << " " << author.m_name << " ]";
return os.str();
}
UID m_uid;
UID m_uid{};
std::string m_name;
};
struct Text
{
Text(std::string const & text, uint8_t const lang)
: m_text(text)
, m_lang(lang)
Text() = default;
Text(std::string const & text, uint8_t const lang) : m_text(text), m_lang(lang) {}
bool operator==(Text const & rhs) const { return m_lang == rhs.m_lang && m_text == rhs.m_text; }
friend std::string DebugPrint(Text const & text)
{
std::ostringstream os;
os << "Text [ " << StringUtf8Multilang::GetLangByCode(text.m_lang) << ": " << text.m_text
<< " ]";
return os.str();
}
std::string m_text;
uint8_t m_lang;
uint8_t m_lang = StringUtf8Multilang::kDefaultCode;
};
struct Review
{
using ReviewId = uint32_t;
using Time = std::chrono::time_point<std::chrono::system_clock>;
Review(Text const & text, Author const & author,
float const rating, bool const evaluation,
std::chrono::time_point<std::chrono::system_clock> const & time)
: m_text(text)
, m_author(author)
, m_rating(rating)
, m_evaluation(evaluation)
, m_time(time)
enum class Sentiment
{
Positive,
Negative
};
Review() = default;
Review(ReviewId id, Text const & text, Author const & author, float const rating,
Sentiment const sentiment, Time const & time)
: m_text(text), m_author(author), m_rating(rating), m_sentiment(sentiment), m_time(time)
{
}
ReviewId m_id;
bool operator==(Review const & rhs) const
{
return m_id == rhs.m_id && m_text == rhs.m_text && m_author == rhs.m_author &&
m_rating == rhs.m_rating && m_sentiment == rhs.m_sentiment && m_time == rhs.m_time;
}
Text m_text;
Author m_author;
uint32_t DaysSinceEpoch() const
{
auto const hours = std::chrono::duration_cast<std::chrono::hours>(m_time.time_since_epoch());
return static_cast<uint32_t>(hours.count()) / 24;
}
void SetDaysSinceEpoch(uint32_t days)
{
auto const hours = std::chrono::hours(days * 24);
m_time = Time(hours);
}
friend std::string DebugPrint(Sentiment const sentiment)
{
switch (sentiment)
{
case Sentiment::Positive: return "Positive";
case Sentiment::Negative: return "Negative";
}
}
friend std::string DebugPrint(Review const & review)
{
std::ostringstream os;
os << "Review [ ";
os << "id:" << review.m_id << ", ";
os << "text:" << DebugPrint(review.m_text) << ", ";
os << "author:" << DebugPrint(review.m_author) << ", ";
os << "rating:" << review.m_rating << ", ";
os << "sentiment:" << DebugPrint(review.m_sentiment) << ", ";
os << "days since epoch:" << review.DaysSinceEpoch() << " ]";
return os.str();
}
ReviewId m_id{};
Text m_text{};
Author m_author{};
// A rating of a review itself. It is accumulated from other users
// likes or dislakes.
float m_rating;
// A positive/negative evaluation given to a place by an author.
bool m_evaluation;
std::chrono::time_point<std::chrono::system_clock> m_time;
// likes or dislikes.
float m_rating{};
// A positive/negative sentiment given to a place by an author.
Sentiment m_sentiment = Sentiment::Positive;
Time m_time{};
};
struct Attribute
{
Attribute() = default;
Attribute(TranslationKey const & key, TranslationKey const & value) : m_key(key), m_value(value)
{
}
TranslationKey m_key;
TranslationKey m_value;
};
bool operator==(Attribute const & rhs) const
{
return m_key == rhs.m_key && m_value == rhs.m_value;
}
// struct Media
// {
// std::unique_ptr<MediaImpl> m_data;
// };
friend std::string DebugPrint(Attribute const & attribute)
{
std::ostringstream os;
os << "Attribute [ key:" << attribute.m_key << ", value:" << attribute.m_value << " ]";
return os.str();
}
TranslationKey m_key{};
TranslationKey m_value{};
};
struct UGC
{
UGC(Rating const & rating,
std::vector<Review> const & reviews,
UGC() = default;
UGC(Rating const & rating, std::vector<Review> const & reviews,
std::vector<Attribute> const & attributes)
: m_rating(rating)
, m_reviews(reviews)
, m_attributes(attributes)
: m_rating(rating), m_reviews(reviews), m_attributes(attributes)
{
}
Rating m_rating;
bool operator==(UGC const & rhs) const {
return m_rating == rhs.m_rating && m_reviews == rhs.m_reviews &&
m_attributes == rhs.m_attributes;
}
friend std::string DebugPrint(UGC const & ugc)
{
std::ostringstream os;
os << "UGC [ ";
os << "rating:" << DebugPrint(ugc.m_rating) << ", ";
os << "reviews:" << DebugPrint(ugc.m_reviews) << ", ";
os << "attributes:" << DebugPrint(ugc.m_attributes) << " ]";
return os.str();
}
Rating m_rating{};
std::vector<Review> m_reviews;
std::vector<Attribute> m_attributes;
// Media m_media;
};
struct ReviewFeedback
{
ReviewFeedback(bool const evaluation,
std::chrono::time_point<std::chrono::system_clock> const time)
: m_evaluation(evaluation)
, m_time(time)
: m_evaluation(evaluation), m_time(time)
{
}
@ -162,12 +267,8 @@ struct ReviewAbuse
struct UGCUpdate
{
UGCUpdate(Rating ratings, Attribute attribute,
ReviewAbuse abuses, ReviewFeedback feedbacks)
: m_ratings(ratings)
, m_attribute(attribute)
, m_abuses(abuses)
, m_feedbacks(feedbacks)
UGCUpdate(Rating ratings, Attribute attribute, ReviewAbuse abuses, ReviewFeedback feedbacks)
: m_ratings(ratings), m_attribute(attribute), m_abuses(abuses), m_feedbacks(feedbacks)
{
}

View file

@ -0,0 +1,10 @@
project(ugc_tests)
set(
SRC
serdes_tests.cpp
)
omim_add_test(${PROJECT_NAME} ${SRC})
omim_link_libraries(${PROJECT_NAME} platform coding geometry base)
link_qt5_core(${PROJECT_NAME})

View file

@ -0,0 +1,117 @@
#include "testing/testing.hpp"
#include "ugc/serdes.hpp"
#include "ugc/types.hpp"
#include "coding/reader.hpp"
#include "coding/writer.hpp"
#include <chrono>
#include <cstdint>
#include <vector>
using namespace std;
using namespace ugc;
namespace
{
using Buffer = vector<uint8_t>;
using Ser = Serializer<MemWriter<Buffer>>;
using Des = DeserializerV0<ReaderSource<MemReader>>;
chrono::hours FromDays(uint32_t days)
{
return std::chrono::hours(days * 24);
}
Rating GetTestRating()
{
vector<RatingRecord> records;
records.emplace_back("music" /* key */, 5.0 /* value */);
records.emplace_back("service" /* key */, 4.0 /* value */);
return Rating(records, 4.5 /* aggValue */);
}
UGC GetTestUGC()
{
Rating rating;
rating.m_ratings.emplace_back("food" /* key */, 4.0 /* value */);
rating.m_ratings.emplace_back("service" /* key */, 5.0 /* value */);
rating.m_ratings.emplace_back("music" /* key */, 5.0 /* value */);
rating.m_aggValue = 4.5;
vector<Review> reviews;
reviews.emplace_back(20 /* id */, Text("Damn good coffee", StringUtf8Multilang::kEnglishCode),
Author(UID(987654321 /* hi */, 123456789 /* lo */), "Cole"),
5.0 /* rating */, Review::Sentiment::Positive,
Review::Time(FromDays(10)));
reviews.emplace_back(67812 /* id */,
Text("Clean place, reasonably priced", StringUtf8Multilang::kDefaultCode),
Author(UID(0 /* hi */, 315 /* lo */), "Cooper"), 5.0 /* rating */,
Review::Sentiment::Positive, Review::Time(FromDays(1)));
vector<Attribute> attributes;
attributes.emplace_back("best-drink", "Coffee");
return UGC(rating, reviews, attributes);
}
MemWriter<Buffer> MakeSink(Buffer & buffer)
{
return MemWriter<Buffer>(buffer);
}
ReaderSource<MemReader> MakeSource(Buffer const & buffer)
{
MemReader reader(buffer.data(), buffer.size());
return ReaderSource<MemReader>(reader);
}
UNIT_TEST(SerDes_Rating)
{
auto const expectedRating = GetTestRating();
TEST_EQUAL(expectedRating, expectedRating, ());
HeaderV0 header;
Buffer buffer;
{
auto sink = MakeSink(buffer);
Ser(sink, header)(expectedRating);
}
Rating actualRating({} /* ratings */, {} /* aggValue */);
{
auto source = MakeSource(buffer);
Des(source, header)(actualRating);
}
TEST_EQUAL(expectedRating, actualRating, ());
}
UNIT_TEST(SerDes_UGC)
{
auto const expectedUGC = GetTestUGC();
TEST_EQUAL(expectedUGC, expectedUGC, ());
HeaderV0 header;
Buffer buffer;
{
auto sink = MakeSink(buffer);
Ser(sink, header)(expectedUGC);
}
UGC actualUGC({} /* rating */, {} /* reviews */, {} /* attributes */);
{
auto source = MakeSource(buffer);
Des(source, header)(actualUGC);
}
TEST_EQUAL(expectedUGC, actualUGC, ());
}
} // namespace