[ugc] Deserialization + combining with serialization. And review fixes

This commit is contained in:
Arsentiy Milchakov 2017-10-05 11:27:00 +03:00 committed by Yuri Gorshenin
parent a316f65f1a
commit 385f1affeb
20 changed files with 208 additions and 177 deletions

View file

@ -38,11 +38,6 @@ void FromJSON(json_t * root, bool & result)
result = json_is_true(root);
}
bool CheckJsonArray(json_t const * data)
{
return data != nullptr && json_is_array(data) && json_array_size(data) > 0;
}
namespace std
{
void FromJSON(json_t * root, string & result)

View file

@ -171,8 +171,6 @@ void FromJSONObjectOptionalField(json_t * root, std::string const & field, std::
FromJSON(json_array_get(arr, i), result[i]);
}
bool CheckJsonArray(json_t const * data);
struct JSONFreeDeleter
{
void operator()(char * buffer) const { free(buffer); }

View file

@ -74,7 +74,7 @@ public:
try
{
FileReader reader(filename);
ReaderSource<FileReader> src(reader);
NonOwningReaderSource src(reader);
Read(src);
}
catch (FileReader::Exception const & e)

View file

@ -6,12 +6,14 @@
#include "ugc/types.hpp"
#include "base/math.hpp"
std::string g_database(R"LLL(
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE ratings (key bigint, value blob);
INSERT INTO "ratings" VALUES(9826352,'{"osm_id":9826352,"total_rating":10.34,"votes":721,"ratings":[{"id":2,"value":3.4},{"id":2,"value":6.0001}],"reviews":[{"id":7864532,"text":"The best service on the Earth","lang":"en","author":"Robert","rating":8.5,"date":1234567}]}');
INSERT INTO "ratings" VALUES(9826353,'{"osm_id":9826353,"total_rating":0.34,"votes":1,"ratings":[{"id":2,"value":3.4},{"id":3,"value":6.0001},{"id":6,"value":0.0001}],"reviews":[{"id":78645323924,"text":"Заебись!","lang":"ru","author":"Вася","rating":10,"date":1234569}]}');
INSERT INTO "ratings" VALUES(9826353,'{"osm_id":9826353,"total_rating":0.34,"votes":1,"ratings":[{"id":2,"value":3.4},{"id":3,"value":6.0001},{"id":6,"value":0.0001}],"reviews":[{"id":78645323924,"text":"Изумительно!","lang":"ru","author":"Вася","rating":10,"date":1234569}]}');
CREATE INDEX key_index ON ratings (key);
COMMIT;
)LLL");
@ -41,5 +43,5 @@ UNIT_TEST(UGC_TranslateRatingTest)
TEST_EQUAL(ugc.m_ratings.size(), 2, ());
TEST_EQUAL(ugc.m_ratings[0].m_key, "2", ());
TEST_LESS(ugc.m_ratings[0].m_value - 3.4, 1e-6, ());
TEST(my::AlmostEqualAbs(ugc.m_ratings[0].m_value, 3.4f, 1e-6f), ());
}

View file

@ -2,59 +2,14 @@
#include "generator/ugc_db.hpp"
#include "ugc/serdes_json.hpp"
#include "coding/multilang_utf8_string.hpp"
#include "base/string_utils.hpp"
#include "3party/jansson/myjansson.hpp"
namespace
{
void FillRatings(json_t const * ratings, ugc::Ratings & result)
{
size_t size = json_array_size(ratings);
for (size_t i = 0; i < size; ++i)
{
json_t * el = json_array_get(ratings, i);
uint32_t id = 0;
double ratingValue = 0.;
FromJSONObject(el, "id", id);
FromJSONObject(el, "value", ratingValue);
result.emplace_back(strings::to_string(id), static_cast<float>(ratingValue));
}
}
void FillReviews(json_t const * reviews, ugc::Reviews & result)
{
size_t size = json_array_size(reviews);
for (size_t i = 0; i < size; ++i)
{
ugc::Review review;
json_t * el = json_array_get(reviews, i);
uint32_t daysSinceEpoch = 0;
double rating = 0.;
std::string lang;
FromJSONObject(el, "id", review.m_id);
FromJSONObject(el, "text", review.m_text.m_text);
FromJSONObject(el, "lang", lang);
FromJSONObject(el, "author", review.m_author);
FromJSONObject(el, "rating", rating);
FromJSONObject(el, "date", daysSinceEpoch);
review.m_text.m_lang = StringUtf8Multilang::GetLangIndex(lang);
review.m_rating = rating;
review.m_time = ugc::Clock::now() - std::chrono::hours(daysSinceEpoch * 24);
result.push_back(std::move(review));
}
}
} // namespace
namespace generator
{
UGCTranslator::UGCTranslator() : m_db(":memory:") {}
@ -64,43 +19,21 @@ UGCTranslator::UGCTranslator(std::string const & dbFilename) : m_db(dbFilename)
bool UGCTranslator::TranslateUGC(osm::Id const & id, ugc::UGC & ugc)
{
std::vector<uint8_t> blob;
bool rc = m_db.Get(id, blob);
if (!rc)
if (!m_db.Get(id, blob))
return false;
std::string result(blob.cbegin(), blob.cend());
try
{
my::Json root(result);
double totalRating = 0.;
std::string src(blob.cbegin(), blob.cend());
FromJSONObject(root.get(), "total_rating", totalRating);
FromJSONObject(root.get(), "votes", ugc.m_votes);
ugc::DeserializerJsonV0 des(src);
ugc.m_totalRating = totalRating;
auto const ratings = json_object_get(root.get(), "ratings");
auto const reviews = json_object_get(root.get(), "reviews");
if (!CheckJsonArray(ratings) || !CheckJsonArray(reviews))
return false;
FillRatings(ratings, ugc.m_ratings);
FillReviews(reviews, ugc.m_reviews);
}
catch (my::Json::Exception const & e)
{
LOG(LERROR, (e.Msg()));
ugc = {};
return false;
}
des(ugc);
return true;
}
void UGCTranslator::CreateDb(std::string const & data)
{
bool rc = m_db.Exec(data);
UNUSED_VALUE(rc);
CHECK(m_db.Exec(data), ());
}
} // namespace generator

View file

@ -37,8 +37,6 @@ void ParseLocals(std::string const & src, std::vector<LocalExpert> & locals,
hasPrevious = json_is_number(previousField);
hasNext = json_is_number(nextField);
auto const results = json_object_get(root.get(), "results");
if (!CheckJsonArray(results))
return;
auto const dataSize = json_array_size(results);
for (size_t i = 0; i < dataSize; ++i)
{

View file

@ -133,7 +133,7 @@ bool CheckAnswer(my::Json const & root)
std::string errorMessage = "Unknown error.";
auto const errorMessageArray = json_object_get(root.get(), "errorMessageText");
if (CheckJsonArray(errorMessageArray))
if (json_array_size(errorMessageArray))
FromJSON(json_array_get(errorMessageArray, 0), errorMessage);
LOG(LWARNING, ("Viator retrieved unsuccessfull status, error message:", errorMessage));
@ -148,7 +148,7 @@ void MakeProducts(std::string const & src, std::vector<Product> & products)
my::Json root(src.c_str());
auto const data = json_object_get(root.get(), "data");
if (!CheckAnswer(root) || !CheckJsonArray(data))
if (!CheckAnswer(root) || !json_array_size(data))
return;
auto const dataSize = json_array_size(data);

View file

@ -1,5 +1,7 @@
#include "ugc/api.hpp"
#include "indexer/feature.hpp"
#include "platform/platform.hpp"
#include <chrono>
@ -9,7 +11,11 @@ using namespace ugc;
namespace ugc
{
Api::Api(Index const & index, std::string const & filename) : m_index(index), m_storage(filename) {}
Api::Api(Index const & index, std::string const & filename)
: m_storage(filename)
, m_loader(index)
{
}
void Api::GetUGC(FeatureID const & id, UGCCallback callback)
{
@ -23,17 +29,18 @@ void Api::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc)
void Api::GetUGCImpl(FeatureID const & id, UGCCallback callback)
{
// TODO (@y, @mgsergio): retrieve static UGC
UGC ugc;
UGCUpdate update;
if (!id.IsValid())
{
GetPlatform().RunOnGuiThread([ugc, update, callback] { callback(ugc, update); });
GetPlatform().RunOnGuiThread([callback] { callback({}, {}); });
return;
}
// ugc = MakeTestUGC1();
UGC ugc;
UGCUpdate update;
m_storage.GetUGCUpdate(id, update);
m_loader.GetUGC(id, ugc);
GetPlatform().RunOnGuiThread([ugc, update, callback] { callback(ugc, update); });
}

View file

@ -2,6 +2,7 @@
#include "base/worker_thread.hpp"
#include "ugc/loader.hpp"
#include "ugc/storage.hpp"
#include "ugc/types.hpp"
@ -28,8 +29,8 @@ private:
void SetUGCUpdateImpl(FeatureID const & id, UGCUpdate const & ugc);
Index const & m_index;
base::WorkerThread m_thread;
Storage m_storage;
Loader m_loader;
};
} // namespace ugc

View file

@ -15,6 +15,8 @@ class BaseCollector
public:
virtual ~BaseCollector() = default;
void VisitVarUint(uint32_t, char const * /* name */ = nullptr) {}
void VisitVarUint(uint64_t, char const * /* name */ = nullptr) {}
virtual void VisitRating(float const f, char const * /* name */ = nullptr) {}
virtual void operator()(string const & /* s */, char const * /* name */ = nullptr) {}
virtual void operator()(Sentiment const /* sentiment */, char const * /* name */ = nullptr) {}

34
ugc/loader.cpp Normal file
View file

@ -0,0 +1,34 @@
#include "ugc/loader.hpp"
#include "ugc/types.hpp"
#include "indexer/feature.hpp"
#include "indexer/index.hpp"
#include "defines.hpp"
namespace ugc
{
Loader::Loader(Index const & index) : m_index(index) {}
void Loader::GetUGC(FeatureID const & featureId, UGC & result)
{
UGC ugc;
auto const & handle = m_index.GetMwmHandleById(featureId.m_mwmId);
if (!handle.IsAlive())
return;
auto const & value = *handle.GetValue<MwmValue>();
if (!value.m_cont.IsExist(UGC_FILE_TAG))
return;
auto readerPtr = value.m_cont.GetReader(UGC_FILE_TAG);
if (!m_d.Deserialize(*readerPtr.GetPtr(), featureId.m_index, ugc))
return;
result = std::move(ugc);
}
} // namespace ugc

22
ugc/loader.hpp Normal file
View file

@ -0,0 +1,22 @@
#pragma once
#include "ugc/binary/serdes.hpp"
class Index;
struct FeatureID;
namespace ugc
{
struct UGC;
class Loader
{
public:
Loader(Index const & index);
void GetUGC(FeatureID const & featureId, UGC & ugc);
private:
Index const & m_index;
binary::UGCDeserializer m_d;
};
} // namespace ugc

View file

@ -73,6 +73,12 @@ public:
ToJSONObject(*m_json, name, d);
}
template <typename T>
void VisitVarUint(T const & t, char const * name = nullptr)
{
ToJSONObject(*m_json, name, t);
}
private:
template <typename Fn>
void NewScopeWith(my::JSONPtr json_object, char const * name, Fn && fn)
@ -95,20 +101,28 @@ private:
Sink & m_sink;
};
template <typename Source>
class DeserializerJsonV0
{
public:
DECLARE_EXCEPTION(Exception, RootException);
DeserializerJsonV0(Source & source) : m_source(source)
template <typename Source,
typename std::enable_if<
!std::is_convertible<Source, std::string>::value, Source>::type * = nullptr>
DeserializerJsonV0(Source & source)
{
std::string src(source.Size(), '\0');
source.Read(static_cast<void *>(&src[0]), source.Size());
m_jsonObject.ParseFrom(src.c_str());
m_jsonObject.ParseFrom(src);
m_json = m_jsonObject.get();
}
DeserializerJsonV0(std::string const & source)
: m_jsonObject(source)
, m_json(m_jsonObject.get())
{
}
void operator()(bool & d, char const * name = nullptr) { FromJSONObject(m_json, name, d); }
void operator()(uint8_t & d, char const * name = nullptr) { FromJSONObject(m_json, name, d); }
void operator()(uint32_t & d, char const * name = nullptr) { FromJSONObject(m_json, name, d); }
@ -118,6 +132,7 @@ public:
{
(*this)(key.m_key, name);
}
void operator()(Time & t, char const * name = nullptr)
{
uint32_t d = 0;
@ -167,6 +182,12 @@ public:
f = static_cast<float>(d);
}
template <typename T>
void VisitVarUint(T & t, char const * name = nullptr)
{
FromJSONObject(m_json, name, t);
}
private:
json_t * SaveContext(char const * name = nullptr)
{
@ -184,6 +205,5 @@ private:
my::Json m_jsonObject;
json_t * m_json = nullptr;
Source & m_source;
};
} // namespace ugc

View file

@ -10,12 +10,13 @@ Storage::Storage(std::string const & filename)
Load();
}
UGCUpdate const * Storage::GetUGCUpdate(FeatureID const & id) const
void Storage::GetUGCUpdate(FeatureID const & id, UGCUpdate & ugc) const
{
auto const it = m_ugc.find(id);
if (it != end(m_ugc))
return &it->second;
return nullptr;
if (it == end(m_ugc))
return;
ugc = it->second;
}
void Storage::SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc)

View file

@ -14,7 +14,7 @@ class Storage
public:
explicit Storage(std::string const & filename);
UGCUpdate const * GetUGCUpdate(FeatureID const & id) const;
void GetUGCUpdate(FeatureID const & id, UGCUpdate & ugc) const;
void SetUGCUpdate(FeatureID const & id, UGCUpdate const & ugc);
void Save();

View file

@ -4,6 +4,7 @@
#include "coding/hex.hpp"
#include "base/math.hpp"
#include "base/visitor.hpp"
#include <chrono>
@ -30,14 +31,14 @@ struct TranslationKey
bool operator==(TranslationKey const & rhs) const { return m_key == rhs.m_key; }
bool operator<(TranslationKey const & rhs) const { return m_key < rhs.m_key; }
friend std::string DebugPrint(TranslationKey const & key)
{
return "TranslationKey [ " + key.m_key + " ]";
}
std::string m_key;
};
std::string DebugPrint(TranslationKey const & key)
{
return "TranslationKey [ " + key.m_key + " ]";
}
enum class Sentiment
{
Positive,
@ -86,18 +87,18 @@ struct RatingRecord
return m_key == rhs.m_key && m_value == rhs.m_value;
}
friend std::string DebugPrint(RatingRecord const & ratingRecord)
{
std::ostringstream os;
os << "RatingRecord [ " << DebugPrint(ratingRecord.m_key) << " " << ratingRecord.m_value
<< " ]";
return os.str();
}
TranslationKey m_key{};
float m_value{};
};
std::string DebugPrint(RatingRecord const & ratingRecord)
{
std::ostringstream os;
os << "RatingRecord [ " << DebugPrint(ratingRecord.m_key) << " " << ratingRecord.m_value
<< " ]";
return os.str();
}
using Ratings = std::vector<RatingRecord>;
struct UID
@ -110,17 +111,18 @@ struct UID
DECLARE_VISITOR(visitor(m_hi, "hi"), visitor(m_lo, "lo"));
bool operator==(UID const & rhs) const { return m_hi == rhs.m_hi && m_lo == rhs.m_lo; }
friend std::string DebugPrint(UID const & uid)
{
std::ostringstream os;
os << "UID [ " << uid.ToString() << " ]";
return os.str();
}
uint64_t m_hi{};
uint64_t m_lo{};
};
std::string DebugPrint(UID const & uid)
{
std::ostringstream os;
os << "UID [ " << uid.ToString() << " ]";
return os.str();
}
using Author = std::string;
struct Text
@ -132,18 +134,18 @@ struct Text
bool operator==(Text const & rhs) const { return m_lang == rhs.m_lang && m_text == rhs.m_text; }
friend std::string DebugPrint(Text const & text)
{
std::ostringstream os;
os << "Text [ " << StringUtf8Multilang::GetLangByCode(text.m_lang) << ": " << text.m_text
<< " ]";
return os.str();
}
std::string m_text;
uint8_t m_lang = StringUtf8Multilang::kDefaultCode;
};
std::string DebugPrint(Text const & text)
{
std::ostringstream os;
os << "Text [ " << StringUtf8Multilang::GetLangByCode(text.m_lang) << ": " << text.m_text
<< " ]";
return os.str();
}
struct Review
{
using ReviewId = uint64_t;
@ -164,18 +166,6 @@ struct Review
m_rating == rhs.m_rating && m_time == rhs.m_time;
}
friend std::string DebugPrint(Review const & review)
{
std::ostringstream os;
os << "Review [ ";
os << "id:" << review.m_id << ", ";
os << "text:" << DebugPrint(review.m_text) << ", ";
os << "author:" << review.m_author << ", ";
os << "rating:" << review.m_rating << ", ";
os << "days since epoch:" << ToDaysSinceEpoch(review.m_time) << " ]";
return os.str();
}
ReviewId m_id{};
Text m_text{};
Author m_author{};
@ -183,6 +173,18 @@ struct Review
Time m_time{};
};
std::string DebugPrint(Review const & review)
{
std::ostringstream os;
os << "Review [ ";
os << "id:" << review.m_id << ", ";
os << "text:" << DebugPrint(review.m_text) << ", ";
os << "author:" << review.m_author << ", ";
os << "rating:" << review.m_rating << ", ";
os << "days since epoch:" << ToDaysSinceEpoch(review.m_time) << " ]";
return os.str();
}
using Reviews = std::vector<Review>;
struct Attribute
@ -199,49 +201,56 @@ struct Attribute
return m_key == rhs.m_key && m_value == rhs.m_value;
}
friend std::string DebugPrint(Attribute const & attribute)
{
std::ostringstream os;
os << "Attribute [ key:" << DebugPrint(attribute.m_key)
<< ", value:" << DebugPrint(attribute.m_value) << " ]";
return os.str();
}
TranslationKey m_key{};
TranslationKey m_value{};
};
std::string DebugPrint(Attribute const & attribute)
{
std::ostringstream os;
os << "Attribute [ key:" << DebugPrint(attribute.m_key)
<< ", value:" << DebugPrint(attribute.m_value) << " ]";
return os.str();
}
struct UGC
{
UGC() = default;
UGC(Ratings const & records, Reviews const & reviews, float const totalRating, uint32_t votes)
: m_ratings(records), m_reviews(reviews), m_totalRating(totalRating), m_votes(votes)
UGC(Ratings const & records, Reviews const & reviews, float const totalRating, uint32_t basedOn)
: m_ratings(records), m_reviews(reviews), m_totalRating(totalRating), m_basedOn(basedOn)
{
}
DECLARE_VISITOR(visitor(m_ratings, "ratings"), visitor(m_reviews, "reviews"),
visitor.VisitRating(m_totalRating, "totalRating"))
visitor.VisitRating(m_totalRating, "total_rating"),
visitor.VisitVarUint(m_basedOn, "based_on"))
bool operator==(UGC const & rhs) const
{
return m_ratings == rhs.m_ratings && m_reviews == rhs.m_reviews;
return m_ratings == rhs.m_ratings && m_reviews == rhs.m_reviews &&
my::AlmostEqualAbs(m_totalRating, rhs.m_totalRating, 1e-6f) && m_basedOn == rhs.m_basedOn;
}
friend std::string DebugPrint(UGC const & ugc)
bool IsValid() const
{
std::ostringstream os;
os << "UGC [ ";
os << "records:" << ::DebugPrint(ugc.m_ratings) << ", ";
os << "reviews:" << ::DebugPrint(ugc.m_reviews) << " ]";
return os.str();
return (!m_ratings.empty() || !m_reviews.empty()) && m_totalRating > 1e-6 && m_basedOn > 0;
}
Ratings m_ratings;
Reviews m_reviews;
float m_totalRating{};
uint32_t m_votes{};
uint32_t m_basedOn{};
};
std::string DebugPrint(UGC const & ugc)
{
std::ostringstream os;
os << "UGC [ ";
os << "records:" << ::DebugPrint(ugc.m_ratings) << ", ";
os << "reviews:" << ::DebugPrint(ugc.m_reviews) << " ]";
return os.str();
}
struct UGCUpdate
{
UGCUpdate() = default;
@ -257,14 +266,9 @@ struct UGCUpdate
return m_ratings == rhs.m_ratings && m_text == rhs.m_text && m_time == rhs.m_time;
}
friend std::string DebugPrint(UGCUpdate const & ugcUpdate)
bool IsValid() const
{
std::ostringstream os;
os << "UGCUpdate [ ";
os << "records:" << ::DebugPrint(ugcUpdate.m_ratings) << ", ";
os << "text:" << DebugPrint(ugcUpdate.m_text) << ", ";
os << "days since epoch:" << ToDaysSinceEpoch(ugcUpdate.m_time) << " ]";
return os.str();
return (!m_ratings.empty() || !m_text.m_text.empty()) && m_time != Time();
}
Ratings m_ratings;
@ -272,6 +276,16 @@ struct UGCUpdate
Time m_time{};
};
std::string DebugPrint(UGCUpdate const & ugcUpdate)
{
std::ostringstream os;
os << "UGCUpdate [ ";
os << "records:" << ::DebugPrint(ugcUpdate.m_ratings) << ", ";
os << "text:" << DebugPrint(ugcUpdate.m_text) << ", ";
os << "days since epoch:" << ToDaysSinceEpoch(ugcUpdate.m_time) << " ]";
return os.str();
}
struct ReviewFeedback
{
ReviewFeedback() = default;

View file

@ -15,11 +15,14 @@ HEADERS += \
binary/serdes.hpp \
binary/ugc_holder.hpp \
binary/visitors.hpp \
loader.hpp \
serdes.hpp \
serdes_json.hpp \
storage.hpp \
types.hpp \
SOURCES += \
api.cpp \
binary/serdes.cpp \
loader.cpp \
storage.cpp \

View file

@ -22,7 +22,7 @@ using Buffer = vector<uint8_t>;
using ToBin = Serializer<MemWriter<Buffer>>;
using FromBin = DeserializerV0<ReaderSource<MemReader>>;
using ToJson = SerializerJson<MemWriter<Buffer>>;
using FromJson = DeserializerJsonV0<ReaderSource<MemReader>>;
using FromJson = DeserializerJsonV0;
Ratings GetTestRating()
{

View file

@ -19,13 +19,13 @@ UGC MakeTestUGC1(Time now)
Reviews reviews;
reviews.emplace_back(20 /* id */, Text("Damn good coffee", StringUtf8Multilang::kEnglishCode),
Author(UID(987654321 /* hi */, 123456789 /* lo */), "Cole"),
Author("Cole"),
5.0 /* rating */, FromDaysAgo(now, 10));
reviews.emplace_back(
67812 /* id */, Text("Clean place, reasonably priced", StringUtf8Multilang::kDefaultCode),
Author(UID(0 /* hi */, 315 /* lo */), "Cooper"), 5.0 /* rating */, FromDaysAgo(now, 1));
Author("Cooper"), 5.0 /* rating */, FromDaysAgo(now, 1));
return UGC(records, reviews, 4.5 /* rating */);
return UGC(records, reviews, 4.5 /* rating */, 4000000000 /* votes */);
}
UGC MakeTestUGC2(Time now)
@ -38,9 +38,9 @@ UGC MakeTestUGC2(Time now)
vector<Review> reviews;
reviews.emplace_back(
119 /* id */, Text("This pie's so good it is a crime", StringUtf8Multilang::kDefaultCode),
Author(UID(0 /* hi */, 315 /* lo */), "Cooper"), 5.0 /* rating */, FromDaysAgo(now, 1));
Author("Cooper"), 5.0 /* rating */, FromDaysAgo(now, 1));
return UGC(records, reviews, 5.0 /* rating */);
return UGC(records, reviews, 5.0 /* rating */, 1 /* votes */);
}
UGCUpdate MakeTestUGCUpdate(Time now)

View file

@ -6,5 +6,6 @@ namespace ugc
{
UGC MakeTestUGC1(Time now = Clock::now());
UGC MakeTestUGC2(Time now = Clock::now());
UGCUpdate MakeTestUGCUpdate(Time now = Clock::now());
} // namespace ugc